ubcexplorerscript.py

import json
import os
import re
from operator import itemgetter

## pathnames to the .txt file and .json output directories. All output files will appear in /data/ubc/2020/json-output
## this folder is in the .gitignore so they won't be staged for commit
dirname = os.path.dirname(__file__)
txt_file_directory = os.path.join(dirname, 'data/ubc/2020/courses')
json_output_directory = os.path.join(dirname, 'data/ubc/2020/json-output')

## global variables
courses_array = []
course = {}

## Author:   @MrBenC88
## REQUIRES: ubcalend_txt.py to already have generated the .txt files
## EFFECTS:  parses .txt files generated by ubcalend_txt.py and converts into json objects
## OUTPUT:   output.json
def generate_json_array():
    try:
         newfile = open(json_output_directory + '/' + "output.json", "w")
    except:
        os.mkdir(json_output_directory)
        newfile = open(json_output_directory + '/' + "output.json", "w")

    for file in os.listdir(txt_file_directory):
        f = open(txt_file_directory + '/' + str(file), "r")

        for line in f:
            if len(line) > 3:
                key, value = line.rstrip().split(': ', 1)
                course[key] = value.strip()

            else:
                if course:
                    courses_array.append(course.copy())
                course.clear()
                continue

    newfile.write(json.dumps(courses_array))
    newfile.close() 

## this is used in credits_to_int() and dept_codes_to_json() - don't remove
codes = []


## REQUIRES: jsonOut directory to be present and ubcalend_txt.py to already have generated the .txt files 
##           generate_json_array() must run before this function
## EFFECTS:  if single credit is specified in course["cred"], convert to int, otherwise set course["cred"] to null
## OUTPUT:   output.json (modified)
def credits_to_int():
    f = open(json_output_directory + "/" + "output.json", "r")
    data = json.load(f)
    for course in data:
        if "cred" in course and len(course["cred"]) == 1:
            course["cred"] = int(course["cred"])
        else:
            course["cred"] = None
        codes.append(course["dept"])
    f.close()
    res = open(json_output_directory + "/" + "output.json", "w")
    res.write(json.dumps(data))
    res.close()


## global variables for use in dept_codes_to_json
codeSet = set()
deptJson = {}
deptArray = []

## REQUIRES: jsonOut directory to be present and ubcalend_txt.py to already have generated the .txt files 
##           generate_json_array() must run before this function
## EFFECTS:  creates a depts.json file with list of all deparments in alphabetical order (not for use with UBC Explorer)
## OUTPUT:   depts.json
def dept_codes_to_json():
    for dept in codes:
        codeSet.add(dept)

    array = []
    for code in codeSet:
        array.append(code)
    
    array.sort()

    for dept in array:
        deptJson["dept"] = dept
        deptArray.append(deptJson.copy())
        deptJson.clear()
    newfile = open(json_output_directory + '/' + "depts.json", "w")
    newfile.write(json.dumps(deptArray))


## REQUIRES: jsonOut directory to be present and ubcalend_txt.py to already have generated the .txt files 
##           generate_json_array() must run before this function
## EFFECTS:  parses course["preq"] and course["creq"] fields in output.json and extracts strings with 
##           3+ capital letters and 3 consecutive digits into an array and replaces the course["preq"] and 
##           course["creq"] field with respective arrays
## OUTPUT:   prereqs.json
def parse_prereqs():
    f = open(json_output_directory + "/" + "output.json", "r")
    data = json.load(f)

    for course in data:
        if "preq" in course:
            course["preq"] = re.findall(r'[A-Z]*\s\d{3}[A-Z]*', course["preq"])
        if "creq" in course:
            course["creq"] = re.findall(r'[A-Z]*\s\d{3}[A-Z]*', course["creq"])

    f.close()
    res = open(json_output_directory + "/" + "prereqs.json", "w")
    res.write(json.dumps(data))
    res.close()


## REQUIRES: jsonOut directory to be present and prereqs.json to have been generated 
##           parse_prereqs() must run before this function
## EFFECTS:  for each course, looks for that course code in all of the courses' prerequisite arrays.
##           if match found, add to current course's course["depn"] array
## OUTPUT:   final_courses.json (not used with UBC Explorer)
def get_dependencies():
    f = open(json_output_directory + "/" + "prereqs.json", "r")
    data = json.load(f)

    for course in data:
        for course2 in data:
            if "preq" in course2:
                if course["code"] in course2["preq"]:
                    if not "depn" in course:
                        course["depn"] = []
                    course["depn"].append(course2["code"])

    f.close()
    res = open(json_output_directory + "/" + "final_courses.json", "w")
    res.write(json.dumps(data))
    res.close()


## REQUIRES: jsonOut directory to be present and prereqs.json and depts.json to have been generated 
##           parse_prereqs() must run before this function
##           dept_codes_to_json() must run before this function
## EFFECTS:  generates json array of all departments and their respective course codes 
##           (not used with UBC Explorer)
## OUTPUT:   final_dept_codes.json 
def get_codes_for_depts():
    depts = open(json_output_directory + "/" + "depts.json", "r")
    courses = open(json_output_directory + "/" + "prereqs.json", "r")

    deptData = json.load(depts)
    courseData = json.load(courses)

    result = []

    for dept in deptData:
        newDept = {}
        newDept["dept"] = dept["dept"]
        newDept["courses"] = []
        for course in courseData:
            if course["dept"] == dept["dept"]:
                newDept["courses"].append(course["code"])
        result.append(newDept.copy())
    
    for dept in result:
        codesOnly = []
        for course in dept["courses"]:
            x = course.split(' ')
            codesOnly.append(x[1])
        dept["courses"] = codesOnly

    res = open(json_output_directory + "/" + "final_dept_codes.json", "w")
    res.write(json.dumps(result))
    res.close()


## REQUIRES: jsonOut directory to be present and prereqs.json and depts.json to have been generated 
##           parse_prereqs() must run before this function
##           dept_codes_to_json() must run before this function
## EFFECTS:  generates json array of all course codes (not used with UBC Explorer)
## OUTPUT:   course_codes.json
def generate_course_codes():
    courses = open(json_output_directory + "/" + "prereqs.json", "r")
    data = json.load(courses)
    course_array = []
    for course in data:
        new_course = {}
        new_course["name"] = course["code"]
        course_array.append(new_course.copy())
        new_course.clear()
    
    course_array.sort(key=itemgetter("name"))

    res = open(json_output_directory + "/" + "course_codes.json", "w")
    res.write(json.dumps(course_array))
    res.close()


## REQUIRES: jsonOut directory to be present and prereqs.json and depts.json to have been generated 
##           parse_prereqs() must run before this function
##           dept_codes_to_json() must run before this function
## EFFECTS:  adds a field with all the UBC Courses URLs associated with each course 
## OUTPUT:   with-links.json (this is the one that gets imported to UBC Explorer)
def generage_ubc_courses_links():
    courses = open(json_output_directory + "/" + "final_courses.json", "r")
    data = json.load(courses)
    course_array = []
    for course in data:
        code = course["code"].split(" ")
        course["link"] = "https://courses.students.ubc.ca/cs/courseschedule?pname=subjarea&tname=subj-course&dept=" + code[0] + "&course=" + code[1]
        course_array.append(course.copy())
    
    res = open(json_output_directory + "/" + "with_links.json", "w")
    res.write(json.dumps(course_array))
    res.close()
    

# call to functions (comment any out that you don't want to run)
generate_json_array()
credits_to_int()
dept_codes_to_json()
parse_prereqs()
get_dependencies() 
get_codes_for_depts()
generate_course_codes()
generage_ubc_courses_links() ## this is the final file we use with UBC Explorer