oldPythonSnakeCode.py

import os
from collections import namedtuple
"""
Given the change in functionality of the calls we have, it likely makes sense to develop a few utility functions here, namely one that returns the tuple of dicts of
all the collections to be easily accessed


Effectively, iterate through the resources, dict of ID's to endpoints, then another dict of Call numbers with Dots
stripped to IDs, and a third dict of IDs to JSONs, all wrapped up in a nice tuple, this way I can limit my API calls to
one function for reading, and one function for writing

I'll also want a writeback function that takes in a dict of IDs to new JSONs, and iterates through, with a toggle for overwrite versus return occupied, vs return non exist


For bulk updating, we should take the following procedure

First we need to stop it at d.636 to review how the data comes in exactly so that we can match it.
We will also want a file output listing changed records. this will be useful for manual validation

First
"""
import copy
from dotenv import load_dotenv
import datetime
import json
import re
from asnake.client import ASnakeClient

"""
Authorize is a quick, 1liner for archivessnake to limit interaction with the library where possible.
It returns an authorized session
"""

def log(uri, original, new, result, filename="logfile.csv"):
    file = open(filename, "ab+")

    text = f"\n{datetime.datetime.now()},{uri},{original},{new},{'Success' if result == 200 else result}\n"
    text = text.encode("utf-8", errors ="ignore")

    try:
        file.write(text)

    except UnicodeEncodeError as e:
        error_char = text[e.start:e.end]
        char_code = ord(error_char)
        print(f"Failed to encode character: {error_char}")
        print(f"Character code: {char_code}")

def create_session():
    load_dotenv()
    endpoint = os.getenv("ARCHIVESSPACE_API_ADDRESS")
    if endpoint == "https://staffarchivesspace.lib.rochester.edu/api/":
        if "continue" != input("you are running this script on production, are you sure you want to continue? type 'continue' to continue:"):
            exit()
    user = os.getenv("ARCHIVESSPACE_USERNAME")
    pw = os.getenv("ARCHIVESSPACE_PASSWORD")
    client = ASnakeClient(baseurl=endpoint, username=user, password=pw)
    client.authorize(user, pw)
    return client

def johnawilliams():
    client = create_session()
    data = client.get("/repositories/2/resources/1180").json()
    notes = data["notes"]
    original = copy.deepcopy(data)
    edited = False
    for note_index, note in enumerate(notes):
        if note["jsonmodel_type"] == "note_singlepart":
            for contentIndex in range(len(note["content"])):
                content = note["content"][contentIndex]
                if re.search("\\s*<p>", content):
                    edited = True
                    note["content"][contentIndex] = re.sub("</p>\\s*<p>", "</p>\n<p>", content)


        else: #note multipart
            for subnote_index, subnote in enumerate(note["subnotes"]):
                if "content" in subnote and re.search("\\s*<p>", subnote["content"]):
                    edited = True
                    content = subnote["content"]
                    data["notes"][note_index]["subnotes"][subnote_index]["content"] = re.sub("</p>\\s*<p>", "</p>\n<p>", content)
    if edited:
        result = client.post(f"/repositories/2/resources/1180",
                                 json=data)
        log(f"/repositories/2/resources/1180", original, data,
            result.status_code)



def responsecleanup(response):
    response = response.text
    response = response.strip(" []{}\"\'\n,")
    return response


# TODO: check output of all this and make sure my lists are flattened in both cases, add error handling

"""
Need a dict of stripped callnums to collection IDs
a dict of collection IDs to collection routes
a dict of collection IDs to collection Objects

todo: this code is not likely to end up in the open source tool; it's going to be more schema specific than I'd like, 
and, in the end, will be a bit redundant through genericization
"""
exceptions = ["A.A31", "A.M85", "A.P23", "A.P24", "A.W22", "A.W23", "A.W25",
    "A.W66", "D.122", "D.184", "D.185", "D.231", "D.236", "D.237", "D.258",
    "D.287", "D.307", "D.325", "D.332", "D.346", "D.358", "D.383", "D.386",
    "D.472", "D.48", "D.486", "D.49", "D.500", "D.504", "D.528", "D.541",
    "D.553", "D.58", "D.602", "D.608", "D.612", "D.620", "D.623", "D.624",
    "D.626", "D.640"]


def process_notes_for_resource(client, resource_uri, resource_data):
    """Process notes for a single resource and return whether edits were made"""
    notes = resource_data.get("notes", [])
    original = copy.deepcopy(resource_data)
    edited = False

    for note_index, note in enumerate(notes):
        if note["jsonmodel_type"] == "note_singlepart":
            for contentIndex in range(len(note.get("content", []))):
                content = note["content"][contentIndex]
                if re.search("\\s*<p>", content):
                    edited = True
                    note["content"][contentIndex] = re.sub("\\s*<p>", "<p>", content)
        else: # note multipart
            for subnote_index, subnote in enumerate(note.get("subnotes", [])):
                if "content" in subnote and re.search("\\s*<p>", subnote["content"]):
                    edited = True
                    resource_data["notes"][note_index]["subnotes"][subnote_index]["content"] = re.sub("\\s*<p>", "<p>", subnote["content"])

    if edited:
        result = client.post(resource_uri, json=resource_data)
        print(f"Updated {resource_uri} - Status: {result.status_code}")
        # Uncomment the line below if you have a log function
        # log(resource_uri, original, resource_data, result.status_code)
        return True
    return False

def process_all_resources(repository="All"):
    """Process all resource records to clean up paragraph tags"""
    client = create_session()
    routes, collections, callNums = getCollectionsByIdentifier(repository)

    total_processed = 0
    total_edited = 0

    print(f"Processing {len(collections)} resource records...")

    for resource_id, resource_data in collections.items():
        resource_uri = routes[resource_id]
        total_processed += 1

        try:
            if process_notes_for_resource(client, resource_uri, resource_data):
                total_edited += 1

            # Optional: Add a small delay to avoid overwhelming the server
            # time.sleep(0.1)

        except Exception as e:
            print(f"Error processing {resource_uri}: {str(e)}")
            continue

    print(f"Processing complete! {total_edited} resources edited out of {total_processed} processed.")


#addOffensiveMaterialsStatement()

def getCollectionsByIdentifier(repository="All"):
    session = create_session()
    routes = dict()
    if repository == "All":
        idlist = []
        repolist = session.get("repositories")
        repolist = json.loads(repolist.text)
        for repo in repolist:
            repo = repo["uri"][1:]
            repocollections = responsecleanup(
                session.get(repo + "/resources?all_ids=true")).split(",")
            for collection in repocollections:
                if not collection:
                    continue
                routes.update(
                    {int(collection): repo + "/resources/" + collection})
            idlist = idlist + repocollections
    elif repository == "2,4":
        idlist = []
        repolist = session.get("repositories")
        repolist = json.loads(repolist.text)
        for repo in repolist:
            repo = repo["uri"][1:]
            if not (repo[-1] == "2" or repo[-1] == "4"):
                continue
            repocollections = responsecleanup(
                session.get(repo + "/resources?all_ids=true")).split(",")
            for collection in repocollections:
                if not collection:
                    continue
                routes.update(
                    {int(collection): repo + "/resources/" + collection})
            idlist = idlist + repocollections
    else:
        idlist = responsecleanup(session.get("/repositories/" + str(
            repository) + "/resources?all_ids=true")).split(",")
        for id in idlist:
            if not id:
                continue
            routes.update({int(id): "repositories/" + str(
                repository) + "/resources/" + id})

    collections = dict()
    callNums = dict()
    for id in routes:
        collection = session.get(routes.get(id)).json()
        collections.update({id: collection})
        callNums.update({collection["id_0"].replace(".", ""): id})
    return (routes, collections, callNums)

# TODO: complete
def changeemailaddress(datatuple):
    session = create_session()
    routes = datatuple[0]
    collections = datatuple[1]
    for id in collections:
        collection = collections.get(id)
        collectionorig = collection
        notes = collection["notes"]
        modified = False
        for note in notes:
            if note["jsonmodel_type"] == 'note_multipart':
                for subnote in note["subnotes"]:
                    if "contents" in subnote:
                        if bool(re.search(r'RAREBKS@library\.rochester\.edu', subnote["contents"])):
                            modified = True
                            subnote["contents"] = re.sub(r'RAREBKS@library\.rochester\.edu', 'rbscp@library.rochester.edu', subnote["contents"])
                    if "content" in subnote:
                        if bool(re.search(r'RAREBKS@library\.rochester\.edu', subnote["content"])):
                            modified = True
                            subnote["content"] = re.sub(r'RAREBKS@library\.rochester\.edu',
                                                        'rbscp@library.rochester.edu', subnote["content"])

            if note["jsonmodel_type"] == 'note_singlepart':
                for i in range(len(note["content"])):
                    if bool(re.search(r'RAREBKS@library\.rochester\.edu', note["content"][i])):
                        modified = True
                        note["content"][i] = re.sub(r'RAREBKS@library\.rochester\.edu', 'rbscp@library.rochester.edu', note["content"])
        if modified:
            result = session.post(routes[id], json=collection)
            log(f"/repositories/2/resources/{id}", collectionorig, collection,
                result.status_code)
#changeemailaddress(getCollectionsByIdentifier("2,4"))

def statementonoffensivematerials(datatuple):
    session = create_session()
    routes = datatuple[0]
    collections = datatuple[1]
    for id in collections:
        if id == 1499:
            pass
        modified = False
        collection = collections.get(id)
        collectionorig = collection
        notes = collection["notes"]
        if not notes:
            notes =[]
        NoteContent = "This collection may contain materials, items, or language which researchers may find offensive and objectionable. Researchers with questions or concerns should contact the department at rbscp@library.rochester.edu"
        has_offensive_materials_note = False
        if any(item in datatuple[2] for item in exceptions):
            has_offensive_materials_note = True
            NoteContent = "This collection is known to contain materials, items, or language which researchers may find offensive and objectionable. Researchers with questions or concerns should contact the department at rbscp@library.rochester.edu"

        for note in notes:

            if note.get("subnotes") and note.get("subnotes")[0].get("content") == NoteContent:
                has_offensive_materials_note = True
            if note.get("content") and note.get("content")[0] == NoteContent:
                has_offensive_materials_note = True
        if not has_offensive_materials_note:
            notes.append({'jsonmodel_type': 'note_multipart',
                              'persistent_id': 'aspace_ref5',
                              'label': 'Statement on Offensive Materials', 'type': 'scopecontents',
                              'subnotes': [
                    {'jsonmodel_type': 'note_text',
                     'content': NoteContent,
                     'publish': True}], 'publish': True})
            modified = True

        if modified:
            result = session.post(routes[id], json=collection)
            log(f"/repositories/2/resources/{id}", collectionorig, collection,
                result.status_code)
#statementonoffensivematerials(getCollectionsByIdentifier("2,4"))

def levedits(datatuple):
    modified = open("modified")
    session = create_session()
    routes = datatuple[0]
    collections = datatuple[1]
    for id in collections:
        collection = collections.get(id)
        notes = collection["notes"]
        if not notes:
            notes = []
        hasAccessNote = False
        hasUseNote = False
        hasPreferredCitation = False
        for note in notes:
            if note.get('type') is not None and note.get(
                    'type') == "accessrestrict":
                hasAccessNote = True
            if note.get('type') is not None and note.get(
                    'type') == "userestrict":
                hasUseNote = True
            if note.get('type') is not None and note.get(
                    'type') == "prefercite":
                hasPreferredCitation = True
        if not hasAccessNote:
            notes.append({'jsonmodel_type': 'note_multipart',
                          'label': 'Restrictions on Access',
                          'type': 'accessrestrict', 'rights_restriction': {
                    'local_access_restriction_type': []}, 'subnotes': [
                    {'jsonmodel_type': 'note_text',
                     'content': 'The ' + collection.get(
                         "title") + 'are open for research use. Researchers are advised to contact the Department of Rare Books, Special Collections, and Preservation, River Campus Libraries, University of Rochester prior to visiting. Upon arrival, researchers will also be asked to fill out a registration form and provide photo identification.',
                     'publish': True}], 'publish': True})
        if not hasUseNote:
            notes.append({'jsonmodel_type': 'note_multipart',
                          'label': 'Restrictions on Use',
                          'type': 'userestrict', 'rights_restriction': {
                    'local_access_restriction_type': []}, 'subnotes': [
                    {'jsonmodel_type': 'note_text',
                     'content': 'Reproductions are made upon request but can be subject to restrictions. Permission to publish materials from the collection must currently be requested. It is the researcher\'s obligation to determine and satisfy copyright or other case restrictions when publishing or otherwise distributing materials found in the collections. For more information contact RAREBKS@library.rochester.edu.',
                     'publish': True}], 'publish': True})
        if not hasPreferredCitation:
            notes.append({'jsonmodel_type': 'note_multipart',
                          'persistent_id': 'aspace_ref5',
                          'label': 'Preferred Citation', 'type': 'prefercite',
                          'subnotes': [{'jsonmodel_type': 'note_text',
                                        'content': '[Item title, item date], [' + collection.get(
                                            "title") + '], [' + collection[
                                                       "id_0"] + '], Rare Books, Special Collections, and Preservation, River Campus Libraries, University of Rochester',
                                        'publish': True}], 'publish': True})
        if not (hasAccessNote and hasUseNote and hasPreferredCitation):
            session.post(routes[id], json=collection)
            modified.write(collection["finding_aid_title"])
    modified.close()


def RedirectGeneration():
    routes, collections, callNums = getCollectionsByIdentifier("2,4")
    findingaids = open("rbscp-hm-csv.csv")
    redirects = open("finding-aid-redirects", "x")
    notfound = []
    for line in findingaids:
        line = line.strip()
        id = callNums.get(line)
        if not id:
            notfound += [line]
            continue
        if not collections[id].get("uri"):
            print("URI NOT FOUND" + str(id) + "    " + line)
        redirects.write(
            "RedirectMatch \"/finding-aids/" + line + ".*\" \"https://archives.rochester.edu" +
            collections[id]["uri"] + "\"\n")
        redirects.write(
            "RedirectMatch \"" + line + ".*\" \"https://archives.rochester.edu" +
            collections[id]["uri"] + "\"\n")
    redirects.close()
    print("finished writing redirects file")
    print(str(notfound))


def processEdits(edits, overwrite="always", conditiontype=None,
                 conditionValue=None):
    pass


def editCollection(collectionJson, fieldname, newvalue):
    pass


# levedits(getCollectionsByIdentifier("2,4"))

"""
collectionjson = {}
ids = client.get("repositories/2/archival_objects", params={"all_ids": True})
topcollections = client.get('/repositories/2/top_containers?all_ids=true')
collections = dict()
callNums = dict()
for id in range(len(ids.text[1:-2].split(","))):
    archival_object = client.get("/repositories/2/archival_objects/" + ids.text[1:-2].split(",")[id])
    ao = archival_object.json()
    if len(ao["ancestors"]) == 1 and collections.geft(ao["ancestors"][0]["ref"]) is None:
        firstAncestor = ao["ancestors"][0]["ref"]
        collectionAO = client.get(firstAncestor).json()
        collections.update({firstAncestor: collectionAO})
        callNums.update({collectionAO["ead_id"]: firstAncestor})
        if collectionAO["ead_id"] == "D.636":
            print("hello")

collectionjson["title"] = "This is a successful test"
collectionjson["resource"] = {"ref": collectionjson["uri"]}
newAO = json.dumps(collectionjson)
update_ao = client.post(a,
                        json=collectionjson)  #### Issue is on this line. I'm updating based off IDS and not based off "a"/ ancestor, so i'm editing a child record, not the parent record
print(update_ao)
"""


# RedirectGeneration()



"""
Per Lev; requisite work for bulk updates

1. Loop through items
    2. Loop through notes fields
        3. Loop through subnotes
            4. Find all instances of </p>[\r\n]*<p>
            5. Replace with version of text that goes </p>\n<p>
            6. Update record (ideally do this with all fields at once for performance reasons)
                a. In order to accomplish this, play pass the JSON until you hit the end of loop 2.

"""


def Remove_Redundant_Spaces():
    connection = create_session()
    AOs = connection.get("repositories/2/archival_objects",
                         params={"all_ids": True}).json()
    for AOidx in range(len(AOs)):
        AO = AOs[AOidx]
        Item = connection.get(
            "/repositories/2/archival_objects/" + str(AO)).json()
        original = copy.deepcopy(Item)
        edited = False
        for noteIndex in range(len(Item["notes"])):
            note = Item["notes"][noteIndex]
            if "subnotes" in note:
                for subnoteIndex in range(len(note["subnotes"])):
                    subnote = note["subnotes"][subnoteIndex]
                    if "content" in subnote and re.search(r"</p>", subnote["content"]):
                        edited = True
                        Item["notes"][noteIndex]["subnotes"][subnoteIndex]["content"] = re.sub(r"<p>", r"", subnote["content"])
                        Item["notes"][noteIndex]["subnotes"][subnoteIndex]["content"] = re.sub(r"</p>", "\n", subnote["content"])
            elif "content" in note:
                for contentIndex in range(len(note["content"])):
                    content = note["content"][contentIndex]
                    if re.search("\\s*<p>", content):
                        edited = True
                        note["content"][contentIndex] = re.sub("\\s*<p>", "\n<p>", content)

        if edited:
            result = connection.post(f"/repositories/2/archival_objects/{AO}",
                                     json=Item)
            log(f"/repositories/2/archival_objects/{AO}", original, Item,
                result.status_code)

#agents ->Linked records across repos

def get_agents():
    connection = create_session()
    person_agents = connection.get("/agents/people", params={"all_ids":True}).json()
    family_agents = connection.get("/agents/families", params={"all_ids":True}).json()
    corp_agents = connection.get("/agents/corporate_entities", params={"all_ids":True}).json()
    software_agents = connection.get("/agents/software", params={"all_ids":True}).json()
    agents_records = {}
    for person in person_agents:
        person_agent = connection.get(f"/agents/people/{person}").json()
        agents_records.update({f"/agents/people/{person}":person_agent})
        if person == 861:
            pass

    for family in family_agents:
        family_agent = connection.get(f"/agents/families/{family}").json()
        agents_records.update({f"/agents/families/{family}":family_agent})
        if family == 5:
            pass
    for corp in corp_agents:
        corp_agent = connection.get(f"/agents/corporate_entities/{corp}").json()
        agents_records.update({f"/agents/corporate_entities/{corp}":corp_agent})

    for software in software_agents:
        software_agent = connection.get(f"/agents/software/{software}").json()
        agents_records.update({f"/agents/software/{software}":software_agent})


    with open("agents.csv", "w+") as outfile:
        outfile.write("name|uri|repos used\n")
        for softagent in software_agents:

            name = agents_records[f"/agents/software/{softagent}"]["names"][0]["sort_name"]
            outfile.write(f"{name}|/agents/software/{softagent}|{agents_records[f"/agents/software/{softagent}"]["used_within_repositories"]}\n")
        for corpagent in corp_agents:
            name = agents_records[f"/agents/corporate_entities/{corpagent}"]["names"][0]["sort_name"]
            outfile.write(f"{name}|/agents/corporate_entities/{corpagent}|{agents_records[f"/agents/corporate_entities/{corpagent}"]["used_within_repositories"]}\n")

        for familyagent in family_agents:
            if familyagent == 5:
                pass
            name = agents_records[f"/agents/families/{familyagent}"]["names"][0]["sort_name"]
            outfile.write(f"{name}|/agents/families/{familyagent}|{agents_records[f"/agents/families/{familyagent}"]["used_within_repositories"]}\n")

        for personagent in person_agents:
            name = agents_records[f"/agents/people/{personagent}"]["names"][0]["sort_name"]
            outfile.write(f"{name}|/agents/people/{personagent}|{agents_records[f"/agents/people/{personagent}"]["used_within_repositories"]}\n")


def get_subjects():
    connection = create_session()
    subjects = connection.get("/subjects", params={"page":1})
    subjects= subjects.json()
    first_page = subjects["first_page"]
    last_page = subjects["last_page"]
    with open("subjects.csv", "w+") as outfile:
        for page_number in range(first_page, last_page+1):
            page = connection.get("/subjects", params={"page":page_number})
            page = page.json()
            for subject in page["results"]:
                outfile.write(f"{subject["uri"]}|{subject['title']}\n")
#each repository is a
def get_repository_list():
    connection = create_session()
    repositories = connection.get("/repositories").json()
    Repository = namedtuple("Repository",["name","display_string","uri","id"])
    repos = []
    for repo in repositories:
        repos+= [Repository(repo["name"],repo["display_string"],repo["uri"],repo["uri"].split('/')[-1])]
    return repos
def get_container():
    connection = create_session()
    repositories = get_repository_list()
    for repo in repositories:
        containers = connection.get(f"repositories/{repo.id}/top_containers/search")
        containers = containers.json()
        for container in containers["results"]:
            pass
    containers = connection.get("/subjects", params={"page":1})
    subjects= subjects.json()
    first_page = subjects["first_page"]
    last_page = subjects["last_page"]
    with open("subjects.csv", "w+") as outfile:
        for page_number in range(first_page, last_page+1):
            page = connection.get("/subjects", params={"page":page_number})
            page = page.json()
            for subject in page["results"]:
                outfile.write(f"{subject["uri"]}|{subject['title']}\n")
def get_repository_name():
    connection = create_session()
    subjects = connection.get("/subjects", params={"page":1})
    subjects= subjects.json()
    first_page = subjects["first_page"]
    last_page = subjects["last_page"]
    with open("subjects.csv", "w+") as outfile:
        for page_number in range(first_page, last_page+1):
            page = connection.get("/subjects", params={"page":page_number})
            page = page.json()
            for subject in page["results"]:
                outfile.write(f"{subject["uri"]}|{subject['title']}\n")
def get_resource_record_title():
    connection = create_session()
    subjects = connection.get("/subjects", params={"page":1})
    subjects= subjects.json()
    first_page = subjects["first_page"]
    last_page = subjects["last_page"]
    with open("subjects.csv", "w+") as outfile:
        for page_number in range(first_page, last_page+1):
            page = connection.get("/subjects", params={"page":page_number})
            page = page.json()
            for subject in page["results"]:
                outfile.write(f"{subject["uri"]}|{subject['title']}\n")

get_resource_record_title()
"""
client = create_session()
archival_objects = client.get("/repositories/2/archival_objects", params={"all_ids": True}).json()
ao = archival_objects[0]
ao = client.get("/repositories/2/archival_objects/" + str(ao)).json()
print(ao)
"""