util.py

import glob
import json
import os
import re


LCC_RE = re.compile(r'^[A-HJ-NP-VZ][A-Z]{0,2}\s*\d[^:]*$')
LCC_CLASSES = ['A','B','C','D','E','F','G','H','J','K','L','M','N','P','Q','R','S','T','U','V','Z']
with open('lcc.json', 'r', encoding='utf-8') as lcc_f:
    lcc = json.load(lcc_f)

def get_lcc_subclasses():
    lcc_subclasses = {}
    for lcc_class_entry in lcc:
        subclasses = []
        for subclass in lcc_class_entry['subclasses']:
            subclasses.append(subclass['code'])
        lcc_subclasses[lcc_class_entry['code']] = subclasses
    return lcc_subclasses

LCC_SUBCLASSES = get_lcc_subclasses()

def is_lcc(call_number, doc):
    if LCC_RE.match(call_number) is None:
        return False
    if len(call_number) > 10 and '.' not in call_number:
        return False
    # the only subclasses with 3 letters are in classes D and K
    if len(call_number) > 2 and call_number[0] not in ['D', 'K'] and call_number[:3].isalpha():
        return False
    # E and F classes have no subclass
    if len(call_number) > 1 and call_number[0] in ['E', 'F'] and call_number[1].isalpha():
        return False
    if len(call_number) == 3 and call_number[2] == '\n':
        # this was a generated call number (in the future we'll generate without \n and use clean data to be safe)
        return False
    if (call_number.startswith('CD') and 'format' in doc and 'CD' in doc['format']):
        # this is actually from the Music Library CD Collection
        return False
    lcc_class = call_number[0]
    if call_number[1].isalpha():
        subclass = call_number[1]
    else:
        subclass = ''
    return subclass in LCC_SUBCLASSES[lcc_class]

def has_lcc(doc):
    if 'callnumber-raw' in doc and is_lcc(doc['callnumber-raw'][0], doc):
        return True
    if 'callnumber-label' in doc:
        for cn in doc['callnumber-label']:
            if is_lcc(cn, doc):
                return True
    return False

def get_lcc(doc):
    if 'callnumber-raw' in doc and is_lcc(doc['callnumber-raw'][0], doc):
        return doc['callnumber-raw'][0]
    if 'callnumber-label' in doc:
        for cn in doc['callnumber-label']:
            if is_lcc(cn, doc):
                return cn
    return None

def get_solr_documents(max_files_to_use, from_index=1, enhanced=False):
    file_index = from_index
    if enhanced:
        directory = 'enhanced_data'
    else:
        directory = 'solr_data'
    if enhanced:
        filename = f"{directory}/docs{file_index:03d}.json"
    else:
        filename = f"{directory}/docs{file_index}.json" # TODO: change to use same 03d formatting
    while os.path.exists(filename):
        with open(filename, 'r', encoding='utf-8') as f:
            file_documents = json.load(f)
        yield from file_documents
        if file_index - from_index + 1 >= max_files_to_use:
            break
        file_index = file_index + 1
        if enhanced:
            filename = f"{directory}/docs{file_index:03d}.json"
        else:
            filename = f"{directory}/docs{file_index}.json" # TODO: change to use same 03d formatting

def get_documents(max_files_to_use, from_index=1, with_call_numbers=None):
    if with_call_numbers is None:
        # return all documents, the ones with call numbers first
        yield from get_documents(int(42*max_files_to_use/100), with_call_numbers=True)
        yield from get_documents(int(58*max_files_to_use/100), with_call_numbers=False)
        return
    if with_call_numbers:
        directory = 'shuffled_documents_with_call_numbers'
    else:
        directory = 'shuffled_documents_without_call_numbers'
    file_index = from_index
    filename = f"{directory}/docs{file_index:03d}.json"
    while os.path.exists(filename):
        #print(f"Loading {filename}")
        with open(filename, 'r', encoding='utf-8') as f:
            file_documents = json.load(f)
        yield from file_documents
        if file_index >= max_files_to_use:
            break
        file_index = file_index + 1
        filename = f"{directory}/docs{file_index:03d}.json"

def nb_documents_with_cn():
    return len(glob.glob("shuffled_documents_with_call_numbers/*.json"))

def lcc_class_to_int(c):
    return LCC_CLASSES.index(c)

def int_to_lcc_class(i):
    return LCC_CLASSES[i]

def lcc_subclass_to_int(lcc_class, lcc_subclass):
    subclasses = LCC_SUBCLASSES[lcc_class]
    try:
        return subclasses.index(lcc_subclass)
    except ValueError:
        return -1

def int_to_lcc_subclass(lcc_class, i):
    return LCC_SUBCLASSES[lcc_class][i]

def doc_to_text(doc):
    text = ''
    if 'title' in doc:
        text = doc['title']
    if 'topic' in doc:
        for topic in doc['topic']:
            text = text + ' ' + topic
    if 'language' in doc:
        for language in doc['language']:
            if language != 'Undetermined' and language != 'No linguistic content':
                text = text + ' ' + language
    if 'format' in doc:
        for frmt in doc['format']:
            text = text + ' ' + frmt
    return text