- 
-
-
-req v3.1
-The UBC Prereq Tree
- -
-
- - Email req@eyqs.ca for feedback and suggestions, such as restoring mobile functionality, adding more courses, or adding more universities. -
-- Please use a device with a larger screen to see enough courses to use req. -
-- Please upgrade to a modern web browser and enable JavaScript to use req. -
-
-
-
-
-
-
diff --git a/req.py b/req.py
deleted file mode 100644
index 2390a56..0000000
--- a/req.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-"""
-req v3.1
-Copyright (c) 2016, 2017, 2018 Eugene Y. Q. Shen.
-
-req is free software: you can redistribute it and/or
-modify it under the terms of the GNU General Public License
-as published by the Free Software Foundation, either version
-3 of the License, or (at your option) any later version.
-
-req is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty
-of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-See the GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see http://www.gnu.org/licenses/.
-"""
-
-import os
-import sys
-import git
-import yaml
-
-CONFIG = None
-ROOT_PATH = None
-
-
-def get_config():
- global CONFIG
- if not CONFIG:
- with open(os.path.join(get_root_path(), "config.yaml")) as CONFIGFILE:
- CONFIG = yaml.safe_load(CONFIGFILE)
- return CONFIG
-
-
-def get_root_path():
- global ROOT_PATH
- if not ROOT_PATH:
- path = os.path.dirname(os.path.realpath(__file__))
- repo = git.Repo(path, search_parent_directories=True)
- ROOT_PATH = repo.git.rev_parse("--show-toplevel")
- return ROOT_PATH
-
-
-def get_year(default):
- if len(sys.argv) > 1:
- return sys.argv[1]
- else:
- return default
-
-
-def get_year_path(path, year):
- return os.path.join(get_root_path(), path.replace("$YEAR", str(year)))
-
-
-def make_dirs(path):
- os.makedirs(os.path.dirname(path), exist_ok=True)
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..842d0ff
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,19 @@
+autopep8==1.6.0
+beautifulsoup4==4.10.0
+black==22.1.0
+bs4==0.0.1
+click==8.0.4
+colorama==0.4.4
+distlib==0.3.4
+dnspython==1.16.0
+filelock==3.6.0
+mypy-extensions==0.4.3
+pathspec==0.9.0
+platformdirs==2.5.1
+pycodestyle==2.8.0
+pymongo==3.12.0
+six==1.16.0
+soupsieve==2.3.1
+toml==0.10.2
+tomli==2.0.1
+virtualenv==20.13.1
diff --git a/scraper.py b/scraper.py
new file mode 100644
index 0000000..7acc05c
--- /dev/null
+++ b/scraper.py
@@ -0,0 +1,79 @@
+from typing import List
+from bs4 import BeautifulSoup
+import urllib.request
+import concurrent.futures
+import re
+
+
+class Scraper:
+ def __init__(self) -> None:
+ self.base_url = "https://www.calendar.ubc.ca/vancouver/courses.cfm?page=code"
+ self.courses_url = "https://courses.students.ubc.ca/cs/courseschedule?pname=subjarea&tname=subj-course"
+
+ def __process_dept(self, link) -> List[dict]:
+ result = []
+ html = urllib.request.urlopen(f"{self.base_url}/{link}")
+ soup = BeautifulSoup(html, features="html.parser")
+ courses = soup.find("dl", {"class": "double"})
+
+ new_course = {}
+
+ for details in courses.find_all(["dt", "dd"]):
+ text = details.text
+ split = text.split(" ")
+ if details.name == "dt":
+ new_course["dept"] = split[0]
+ new_course["code"] = f"{split[0]} {split[1]}"
+ new_course["cred"] = self.__credits_to_number(split[2][1:-1]) # (3) -> 3, (1-6) -> 1-6
+ new_course["link"] = f"{self.courses_url}&dept={split[0]}&course={split[1]}"
+ new_course["name"] = details.find("b").text
+ elif details.name == "dd":
+ new_course["desc"] = self.__strip(text.split("\n")[0])
+
+ if "Prerequisite:" in text:
+ preq_text = self.__strip(text.split("Prerequisite:")[1].split("Corequisite:")[0].split("Equivalency:")[0])
+ self.__process_reqs(preq_text, new_course, "prer", "preq")
+
+ if "Corequisite:" in text:
+ creq_text = self.__strip(text.split("Corequisite:")[1].split("Equivalency:")[0])
+ self.__process_reqs(creq_text, new_course, "crer", "creq")
+
+ # skip grad
+ if int(new_course["code"].split(" ")[1]) < 500:
+ result.append(new_course.copy())
+ new_course.clear()
+
+ return result
+
+ def __process_reqs(self, text: str, course: dict, text_label: str, req_label: str) -> List[str]:
+ course[text_label] = text
+ reqs = re.findall(r"\b[A-Z]{3,4} \d{3}\b", text)
+ if len(reqs):
+ course[req_label] = reqs
+
+ def __strip(self, text: str):
+ return re.sub(r"[ ]{2,}", " ", text).strip()
+
+ def __credits_to_number(self, creds: str):
+ if creds.isnumeric():
+ return int(creds)
+ try:
+ num = float(creds)
+ return num
+ except:
+ return None
+
+ def main(self) -> List[dict]:
+ html = urllib.request.urlopen(self.base_url)
+ soup = BeautifulSoup(html, features="html.parser")
+ table = soup.find("table")
+ dept_links = set([e.attrs["href"] for e in table.find_all("a") if "href" in e.attrs])
+
+ result = []
+
+ with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+ results = executor.map(self.__process_dept, list(dept_links))
+ for r in results:
+ result += r
+
+ return result
diff --git a/scrapers/ubc/__pycache__/req.cpython-37.pyc b/scrapers/ubc/__pycache__/req.cpython-37.pyc
deleted file mode 100644
index 697d7b8..0000000
Binary files a/scrapers/ubc/__pycache__/req.cpython-37.pyc and /dev/null differ
diff --git a/scrapers/ubc/excluded.py b/scrapers/ubc/excluded.py
deleted file mode 100644
index f868db7..0000000
--- a/scrapers/ubc/excluded.py
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/usr/bin/env python3
-"""
-req v3.1
-Copyright (c) 2016, 2017, 2018 Eugene Y. Q. Shen.
-
-req is free software: you can redistribute it and/or
-modify it under the terms of the GNU General Public License
-as published by the Free Software Foundation, either version
-3 of the License, or (at your option) any later version.
-
-req is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty
-of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-See the GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see http://www.gnu.org/licenses/.
-"""
-
-import req
-import re
-import bs4
-import requests
-
-CONFIG = req.get_config()["scrapers"]["ubc"]["scripts"]["excluded.py"]
-YEAR = req.get_year(CONFIG['year'])
-OUTFILE = req.get_year_path(CONFIG["outfile"], YEAR)
-
-req.make_dirs(OUTFILE)
-
-
-def parse(words):
- courses = set()
- subject = ''
- for word in re.split('(?:,|or)', words):
- split = word.split()
- if len(split) > 1:
- subject = split[0].strip()
- courses.add(' '.join([subject] + split[-1:]))
- return courses
-
-
-if __name__ == '__main__':
- res = requests.get('http://www.calendar.ubc.ca/' +
- 'vancouver/?tree=12,215,410,414')
- try:
- res.raise_for_status()
- except Exception as exc:
- print('There was a problem:', exc)
- else:
- soup = bs4.BeautifulSoup(res.text, 'html.parser')
-
- with open(OUTFILE, 'w', encoding='utf8') as f:
- for excls in soup.select('ol li'):
- if "following" in excls.contents[0]:
- excluded = parse(excls.contents[0].split('following')[0])
- excluder = parse(excls.contents[0].split('following')[1])
- for code in excluded:
- f.write('\n\ncode: ' + code)
- f.write('\nexcl: ' + ', '.join(excluder))
- else:
- courses = parse(excls.contents[0])
- for code in courses:
- f.write('\n\ncode: ' + code)
- courses.remove(code)
- f.write('\nexcl: ' + ', '.join(courses))
- courses.add(code)
diff --git a/scrapers/ubc/req.py b/scrapers/ubc/req.py
deleted file mode 100644
index 2390a56..0000000
--- a/scrapers/ubc/req.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-"""
-req v3.1
-Copyright (c) 2016, 2017, 2018 Eugene Y. Q. Shen.
-
-req is free software: you can redistribute it and/or
-modify it under the terms of the GNU General Public License
-as published by the Free Software Foundation, either version
-3 of the License, or (at your option) any later version.
-
-req is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty
-of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-See the GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see http://www.gnu.org/licenses/.
-"""
-
-import os
-import sys
-import git
-import yaml
-
-CONFIG = None
-ROOT_PATH = None
-
-
-def get_config():
- global CONFIG
- if not CONFIG:
- with open(os.path.join(get_root_path(), "config.yaml")) as CONFIGFILE:
- CONFIG = yaml.safe_load(CONFIGFILE)
- return CONFIG
-
-
-def get_root_path():
- global ROOT_PATH
- if not ROOT_PATH:
- path = os.path.dirname(os.path.realpath(__file__))
- repo = git.Repo(path, search_parent_directories=True)
- ROOT_PATH = repo.git.rev_parse("--show-toplevel")
- return ROOT_PATH
-
-
-def get_year(default):
- if len(sys.argv) > 1:
- return sys.argv[1]
- else:
- return default
-
-
-def get_year_path(path, year):
- return os.path.join(get_root_path(), path.replace("$YEAR", str(year)))
-
-
-def make_dirs(path):
- os.makedirs(os.path.dirname(path), exist_ok=True)
diff --git a/scrapers/ubc/ubcalend_json.py b/scrapers/ubc/ubcalend_json.py
deleted file mode 100644
index 0ff7f34..0000000
--- a/scrapers/ubc/ubcalend_json.py
+++ /dev/null
@@ -1,236 +0,0 @@
-#!/usr/bin/env python3
-"""
-req v3.1
-Copyright (c) 2016, 2017, 2018 Eugene Y. Q. Shen.
-
-req is free software: you can redistribute it and/or
-modify it under the terms of the GNU General Public License
-as published by the Free Software Foundation, either version
-3 of the License, or (at your option) any later version.
-
-req is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty
-of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-See the GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see http://www.gnu.org/licenses/.
-"""
-
-## This script will parse the data into individual JSON files. You probably don't want to
-## run this script if you're scraping data for UBC Explorer!
-
-import req
-import urllib.request
-import json
-
-CONFIG = req.get_config()['scrapers']['ubc']['scripts']['ubcalend.py']
-YEAR = req.get_year(CONFIG['year'])
-LOGFILE = req.get_year_path(CONFIG['logfile'], YEAR)
-OUTFOLDER = req.get_year_path(CONFIG['outfolder'], YEAR)
-
-req.make_dirs(LOGFILE)
-req.make_dirs(OUTFOLDER)
-
-# use this to initialize dict if we need all fields present
-# dictdata = {
-# "code": "",
-# "name": "",
-# "desc": "",
-# "prer": "",
-# "crer": "",
-# "preq": "",
-# "creq": "",
-# "excl": "",
-# "term": "",
-# "cred": ""
-# }
-
-# Translate a file into a format that req can parse
-def translate(url, outjson):
- dictlist = []
- html = urllib.request.urlopen(url)
- # Skip all lines before the actual course list
- for byte in html:
- line = byte.decode('UTF-8', 'backslashreplace')
- if line.strip() != '- ':
- continue
- break
-
- dictdata = {}
- # Open the output .txt file and start writing to it
- hascode = False # The corresponding course code
- for byte in html:
- line = byte.decode('UTF-8', 'backslashreplace')
- if line.strip() == '
-
- );
- };
-};
-
-ReactDOM.render(
- {constants.tab_list.map((tab) => {
- if (tab == this.state.tab) {
- return (
-
-
-
- {tab}
-
- );
- } else {
- return (
- this.setState({tab})}>
- {tab}
-
- );
- }
- })}
-
- {paragraphs.map((paragraph, index) => {
- return
- );
-};
-
-
-
-export default class Browser extends React.Component {
- constructor(props) {
- // this.props.course_list: the list of course objects
- // this.props.parseCodes: callback for when user updates the course list
- // this.props.updateNeeds: callback for when user clicks the course
- // this.props.updateCourse: callback for when user updates the course
- super(props);
- this.state = {
- render_toggle: false, // toggle this every time you don't want to render
- min_height: 0, // minimum height of the app
- hover_code: "", // course that the user is currently hovering over
- unshade_all: true, // triggered some time after hover_code becomes ""
- unshade_timeout: null,// callback for unshade_all, store it to cancel
- };
- };
-
-
- // callback to update hover_code if the user hovers over a child button
- // child should call updateHover(this.props.code) if the user hovers in
- // and updateHover("") if the user hovers out, to reset it
-
- updateHover(hover_code) {
- if (hover_code === "") {
- this.setState({hover_code, unshade_timeout:
- setTimeout(() => this.setState({unshade_all: true}),
- constants.unshade_delay_ms)});
- } else {
- if (this.state.unshade_timeout) {
- clearTimeout(this.state.unshade_timeout);
- }
- this.setState({hover_code, unshade_all: false, unshade_timeout: null});
- }
- };
-
-
- // update the maximum height after rendering
-
- componentDidUpdate() {
- const min_height = Math.max(this.state.min_height,
- document.getElementById("browser").clientHeight)
- - 2 * remToPixels(constants.sidebar_padding);
- const render_toggle = !this.state.render_toggle;
- this.setState({min_height, render_toggle});
- };
-
-
- // do not re-render if render_toggle has been toggled
-
- shouldComponentUpdate(nextProps, nextState) {
- if (this.state.render_toggle != nextState.render_toggle) {
- return false;
- }
- return true;
- };
-
-
- // draw the entire app only if user selects some courses
-
- render() {
- const button_lists = {};
- for (const code in this.props.course_dict) {
- if (this.props.course_dict.hasOwnProperty(code)) {
- this.props.updateCourse(code);
- const depth = this.props.course_dict[code].depth;
- if (!button_lists[depth]) {
- button_lists[depth] = [];
- }
- let highlighted = false;
- let shaded = !this.state.unshade_all;
- let reqs = "highs";
- if (this.state.hover_code) {
- if (code == this.state.hover_code) {
- shaded = false;
- highlighted = true;
- } else {
- for (const param of ["preqs", "creqs", "excls", "dreqs"]) {
- if (this.props.course_dict[this.state.hover_code]
- [param].includes(code)) {
- shaded = false;
- highlighted = true;
- reqs = param;
- }
- }
- }
- }
- button_lists[depth].push({code, reqs, shaded, highlighted,
- needs: this.props.course_dict[code].needs,
- });
- }
- }
-
- if (Object.keys(button_lists).length === 0) {
- return (
- {paragraph}
- })} -
-
- );
- }
-
- return (
-
-
- );
- };
-};
diff --git a/src/button.jsx b/src/button.jsx
deleted file mode 100644
index 3b2462a..0000000
--- a/src/button.jsx
+++ /dev/null
@@ -1,59 +0,0 @@
-/* req v3.1
- * Copyright (c) 2016, 2017, 2018 Eugene Y. Q. Shen.
- *
- * req is free software: you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation, either version
- * 3 of the License, or (at your option) any later version.
- *
- * req is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see http://www.gnu.org/licenses/.
- */
-import React from "react";
-import * as constants from "./const.js";
-
-function getBackground(needs, shaded) {
- return constants.rgba
- + constants.colours_to_rgba[constants.button_colours[needs]]
- + (shaded ? constants.button_shaded_alpha
- : constants.button_plain_alpha);
-};
-
-function getBorder(reqs, highlighted) {
- return (highlighted ? constants.button_highlight_border
- : constants.button_plain_border)
- + constants.border_colours[reqs];
-};
-
-export default class Button extends React.Component {
- constructor(props) {
- // this.props.code: the course code
- // this.props.reqs: the relationship to the hover_code course
- // this.props.needs: the course status and button colour
- // this.props.shaded: true if the button should be shaded
- // this.props.highlighted: true if the button should be highlighted
- // this.props.updateNeeds: callback for when user clicks the button
- // this.props.updateHover: callback for when user starts/stops hovering
- super(props);
- };
-
- render() {
- return (
-
-
-
- {Object.entries(button_lists).map(([depth, button_list]) => {
- return
-
- this.props.updateNeeds(this.props.code)}
- onMouseOver={() => this.props.updateHover(this.props.code)}
- onMouseOut={() => this.props.updateHover("")}>
- {this.props.code}
-
- );
- };
-};
diff --git a/src/button_row.jsx b/src/button_row.jsx
deleted file mode 100644
index 734244f..0000000
--- a/src/button_row.jsx
+++ /dev/null
@@ -1,46 +0,0 @@
-/* req v3.1
- * Copyright (c) 2016, 2017, 2018 Eugene Y. Q. Shen.
- *
- * req is free software: you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation, either version
- * 3 of the License, or (at your option) any later version.
- *
- * req is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see http://www.gnu.org/licenses/.
- */
-import React from "react";
-import * as constants from "./const.js";
-import Button from "./button.jsx";
-
-export default class ButtonRow extends React.Component {
- constructor(props) {
- // this.props.button_list: the list of course button objects
- // this.props.updateNeeds: callback for when user clicks the course
- // this.props.updateHover: callback for when user starts/stops hovering
- super(props);
- };
-
- render() {
- return (
-
- {this.props.button_list.sort(constants.code_compare).map(
- ({code, reqs, needs, shaded, highlighted}) => {
- return
- })}
-
- );
- };
-};
diff --git a/src/const.js b/src/const.js
deleted file mode 100644
index ab32950..0000000
--- a/src/const.js
+++ /dev/null
@@ -1,161 +0,0 @@
-/* req v3.1
- * Copyright (c) 2016, 2017, 2018 Eugene Y. Q. Shen.
- *
- * req is free software: you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation, either version
- * 3 of the License, or (at your option) any later version.
- *
- * req is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see http://www.gnu.org/licenses/.
- */
-export const rgba = "rgba(";
-export const button_plain_alpha = ", 1)";
-export const button_shaded_alpha = ", 0.3)";
-export const unshade_delay_ms = 1000;
-
-export const button_plain_border = "inset 0px 0px 0px 1px ";
-export const button_highlight_border = "inset 0px 0px 0px 3px ";
-
-export const sidebar_padding = "1rem";
-
-export const tab_list = ["helper", "browser"];
-export const tabbar_style = {
- display: "flex",
- flexDirection: "row",
-};
-export const tab_style = {
- backgroundColor: "whitesmoke",
- padding: "0.3rem 1.2rem",
- border: "1px solid black",
- borderRadius: "10px 10px 0 0",
- borderBottom: "0",
-};
-export const focus_tab_style = {
- ...tab_style,
- backgroundColor: "white",
- position: "relative",
- zIndex: "2",
- bottom: "-1px",
- fontWeight: "bold",
-};
-
-export const wrapper_style = {
- margin: "0 auto",
- width: "80vw",
-};
-
-export const main_style = {
- border: "1px solid black",
-};
-
-export const helper_style = {
- padding: "2rem"
-};
-
-export const forms_style = {
- padding: "3rem",
- display: "flex",
- flexDirection: "row",
-};
-
-export const browser_style = {
- padding: "1rem",
- display: "flex",
- backgroundColor: "rgba(0, 0, 0, 0.5)",
-};
-
-export const button_lists_style = {
- flex: "1",
-};
-
-export const sidebar_style = {
- width: "25rem",
- margin: "0 0 0 0.5rem",
- padding: sidebar_padding,
- backgroundColor: "honeydew",
-};
-
-export const button_style = {
- textAlign: "center",
- width: "100%",
- height: "2rem",
- lineHeight: "2rem",
- cursor: "default",
- transition: "background-color 0.5s, box-shadow 0.5s",
-};
-
-export const button_row_style = {
- display: "grid",
- justifyContent: "center",
- gridTemplateColumns: "repeat(auto-fill, 6rem)",
- gridGap: "0.7rem",
- padding: "1rem 0",
-};
-
-export const button_colours = {
- done: "greenyellow",
- none: "whitesmoke",
- xout: "lavender",
- outs: "wheat",
- creq: "gold",
- preq: "pink",
- excl: "lightsteelblue",
-};
-
-export const button_descriptions = {
- done: " is already taken.",
- none: " can be taken.",
- xout: " can be taken"
- + " unless you've taken some credit excluded course outside the tree.",
- outs: " cannot be taken"
- + " unless you've taken some requisite course outside the tree.",
- creq: " cannot be taken, due to a missing corequisite.",
- preq: " cannot be taken, due to a missing prerequisite.",
- excl: " cannot be taken, due to a"
- + " credit excluded course that you've already taken.",
-};
-
-export const border_colours = {
- highs: "black",
- preqs: "deeppink",
- creqs: "darkorange",
- excls: "indigo",
- dreqs: "olive",
-};
-
-export const border_descriptions = {
- highs: " is the highlighted course.",
- preqs: " is a prerequisite of the highlighted course.",
- creqs: " is a corequisite of the highlighted course.",
- excls: " is credit excluded with the highlighted course.",
- dreqs: " has the highlighted course as a requisite.",
-};
-
-export const colours_to_rgba = {
- greenyellow: "173, 255, 47",
- whitesmoke: "245, 245, 245",
- lavender: "230, 230, 250",
- wheat: "245, 222, 179",
- gold: "255, 215, 0",
- pink: "255, 192, 203",
- lightsteelblue: "176, 196, 222",
- black: "0, 0, 0",
- deeppink: "211, 20, 147",
- darkorange: "255, 140, 0",
- indigo: "75, 0, 130",
- olive: "128, 128, 0",
-};
-
-export function code_compare(a, b) {
- if (a.code < b.code)
- return -1;
- if (a.code > b.code)
- return 1;
- return 0;
-};
diff --git a/src/forms.jsx b/src/forms.jsx
deleted file mode 100644
index ce89ffb..0000000
--- a/src/forms.jsx
+++ /dev/null
@@ -1,124 +0,0 @@
-/* req v3.1
- * Copyright (c) 2016, 2017, 2018 Eugene Y. Q. Shen.
- *
- * req is free software: you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation, either version
- * 3 of the License, or (at your option) any later version.
- *
- * req is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see http://www.gnu.org/licenses/.
- */
-import React from "react";
-import * as constants from "./const.js";
-import course_data from '../req.json';
-
-
-// strip all whitespace characters from a string
-
-function stripWhitespace(string) {
- return string.replace(/\s/g, "");
-};
-
-
-// compile a regular expression from a string
-
-function compileRegExp(string) {
- return new RegExp(string, "i");
-};
-
-
-
-export default class Form extends React.Component {
- constructor(props) {
- // this.props.parseCodes: callback for when user updates the course list
- super(props);
- this.state = {code: ""}
- };
-
-
- // update the course code input box with all selected courses
-
- updateCodes(reqlist) {
- try {
- const re = compileRegExp(stripWhitespace(this.state.code));
- const codes = [];
- for (const code in course_data) {
- if (course_data.hasOwnProperty(code)) {
- const match = re.exec(code);
- if (match !== null) {
- if (reqlist) {
- Array.prototype.push.apply(codes, course_data[code][reqlist]);
- } else {
- codes.push(code);
- }
- }
- }
- }
- if (codes.length > 0) {
- const value = document.getElementById("courses").value.trim();
- if (value.length > 0 && value[value.length - 1] != ",") {
- document.getElementById("courses").value += ", ";
- }
- if (reqlist) {
- for (const code of codes) {
- if (course_data[code][reqlist].length > 0) {
- document.getElementById("courses").value +=
- course_data[code][reqlist].join(", ") + ", ";
- }
- }
- } else {
- for (const code of codes) {
- document.getElementById("courses").value +=
- code + ", ";
- }
- }
- this.setState({code: ""});
- } else {
- this.setState({code: "Error: Not Found"});
- }
- } catch (ignore) {
- this.setState({code: "Error: Invalid Input"});
- }
- };
-
-
- // update all forms with the current state
-
- render() {
- return (
-
-
- );
- };
-};
diff --git a/src/helper.jsx b/src/helper.jsx
deleted file mode 100644
index bd301d5..0000000
--- a/src/helper.jsx
+++ /dev/null
@@ -1,70 +0,0 @@
-/* req v3.1
- * Copyright (c) 2016, 2017, 2018 Eugene Y. Q. Shen.
- *
- * req is free software: you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation, either version
- * 3 of the License, or (at your option) any later version.
- *
- * req is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see http://www.gnu.org/licenses/.
- */
-import React from "react";
-import * as constants from "./const.js";
-
-
-export default class Helper extends React.Component {
- constructor(props) {
- super(props);
- };
-
-
- // draw legend for button and border colours
-
- render() {
- return (
-
-
- this.setState({[e.target.id]: e.target.value})} />
-
-
-
-
-
-
-
-
-
-
-
- );
- };
-};
diff --git a/src/main.jsx b/src/main.jsx
deleted file mode 100644
index 337ab74..0000000
--- a/src/main.jsx
+++ /dev/null
@@ -1,372 +0,0 @@
-/* req v3.1
- * Copyright (c) 2016, 2017, 2018 Eugene Y. Q. Shen.
- *
- * req is free software: you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation, either version
- * 3 of the License, or (at your option) any later version.
- *
- * req is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see http://www.gnu.org/licenses/.
- */
-import "babel-polyfill";
-import React from "react";
-import * as constants from "./const.js";
-import Helper from "./helper.jsx";
-import Browser from "./browser.jsx";
-import course_data from '../req.json';
-
-
-// return a new course object using input data from course_data
-
-function makeCourse(data) {
- return {needs: "none", depth: 0,
- preqs: [], creqs: [], excls: [], dreqs: [], ddict: {}, ...data};
-};
-
-
-// strip all whitespace characters from a string
-
-function stripWhitespace(string) {
- return string.replace(/\s/g, "");
-};
-
-
-// compile a regular expression from a string
-
-function compileRegExp(string) {
- if (string.length == 0) {
- return new RegExp("^$", "i");
- }
- if (string[0] != "^") {
- string = "^" + string;
- }
- if (string[string.length - 1] != "$") {
- string = string + "$";
- }
- return new RegExp(string, "i");
-};
-
-
-// given a list of lists, return a flat list of all valid course codes in it
-
-function flatten(listlist) {
- const flat_list = [];
- for (const list of listlist) {
- if (list instanceof Array) {
- flat_list.push(...flatten(list));
- } else if (course_data.hasOwnProperty(list)) {
- flat_list.push(list);
- }
- }
- return flat_list;
-};
-
-
-
-export default class Main extends React.Component {
-
- // start the application
-
- constructor(props) {
- // this.props.tab: the current tab the user is on
- super(props);
- this.state = {
- course_dict: {}, // course_dict["CPSC 110"] = makeCourse()
- };
-
- // make all course data into course objects
- for (const code in course_data) {
- if (course_data.hasOwnProperty(code)) {
- course_data[code] = makeCourse(course_data[code]);
- }
- }
-
- // update course_data with preqs, creqs, and dreqs
- for (const code in course_data) {
- if (course_data.hasOwnProperty(code)) {
- for (const param of [["excl", "excls", false],
- ["preq", "preqs", true], ["creq", "creqs", true]]) {
- course_data[code][param[1]] = flatten(course_data[code][param[0]])
- if (param[2]) {
- for (const dependency of course_data[code][param[1]]) {
- if (course_data.hasOwnProperty(dependency)) {
- course_data[dependency].ddict[code] = true;
- }
- }
- }
- }
- }
- }
- for (const code in course_data) {
- if (course_data.hasOwnProperty(code)) {
- course_data[code].dreqs = Object.keys(course_data[code].ddict);
- }
- }
- document.getElementById("noscript").style.display = "none";
- };
-
-
- // parse the course code input box and reorder the buttons on the tree
-
- parseCodes() {
- const re_lists = document.getElementById("courses").value.split(";").map(
- (list) => list.split(",").map(
- (code) => compileRegExp(stripWhitespace(code))));
- const new_list = [];
- const done_list = [];
- for (const re of re_lists[0]) {
- for (const code in course_data) {
- if (course_data.hasOwnProperty(code)) {
- const match = re.exec(code);
- if (match !== null) {
- new_list.push(code);
- }
- }
- }
- }
- if (re_lists.length > 1) {
- for (const re of re_lists[1]) {
- for (const code in course_data) {
- if (course_data.hasOwnProperty(code)) {
- const match = re.exec(code);
- if (match !== null) {
- done_list.push(code);
- }
- }
- }
- }
- }
- const code_dict = {};
- for (const code of new_list) {
- if (code.length > 1 && course_data.hasOwnProperty(code)) {
- code_dict[code] = true;
- }
- }
-
- // add all prerequisites and corequisites of all codes, recursively
- const reqlists = ["preqs", "creqs"];
- const checked_dict = {};
- while (Object.keys(checked_dict).length != Object.keys(code_dict).length) {
- for (const code in code_dict) {
- if (code_dict.hasOwnProperty(code)) {
- if (!checked_dict[code]) {
- checked_dict[code] = true;
- for (const reqlist of reqlists) {
- for (const req of course_data[code][reqlist]) {
- if (course_data.hasOwnProperty(req)) {
- code_dict[req] = true;
- }
- }
- }
- }
- }
- }
- }
-
- // create new courses for each course and mark some as done
- const unordered = {};
- const course_dict = {};
- for (const code in code_dict) {
- unordered[code] = true;
- course_dict[code] = makeCourse(course_data[code]);
- if (this.state.course_dict[code]
- && this.state.course_dict[code].needs === "done") {
- course_dict[code].needs = "done";
- }
- }
- if (done_list) {
- for (const code of done_list) {
- if (course_data.hasOwnProperty(code)) {
- unordered[code] = true;
- course_dict[code] = makeCourse(course_data[code]);
- course_dict[code].needs = "done";
- }
- }
- }
-
- /* Arrange courses in order depending on their depth of prereqs
- * First scan through courses with no preqs and set their depth to 1,
- * then scan through all courses whose preqs all have a non-zero depth
- * of which the maximum is 1, and set their depth to 2, etc. until done.
- */
- let depth = 0;
- while (Object.keys(unordered).length !== 0) {
- depth += 1;
- for (const code in unordered) {
- if (unordered.hasOwnProperty(code)) {
- let haspreq = false; // has a prereq in the current tree
- let badpreq = false; // has a prereq with zero or current depth
- let hascreq = false; // has a coreq in the current tree
- let badcreq = false; // has a coreq with zero or current depth
- for (const preq of course_dict[code].preqs) {
- if (course_dict.hasOwnProperty(preq)) {
- haspreq = true;
- if (course_dict[preq].depth === 0
- || course_dict[preq].depth === depth) {
- badpreq = true;
- }
- }
- }
- for (const creq of course_dict[code].creqs) {
- if (course_dict.hasOwnProperty(creq)) {
- hascreq = true;
- if (course_dict[creq].depth === 0
- || course_dict[creq].depth === depth + 0.5) {
- badcreq = true;
- }
- }
- }
- if ((depth === 1 && !haspreq && !hascreq)
- || (!badpreq && !badcreq)) {
- course_dict[code].depth = depth;
- delete unordered[code];
- } else if (!badpreq) {
- course_dict[code].depth = depth + 0.5;
- }
- }
- }
- for (const code in unordered) {
- let badcreq = false; // has a coreq with zero depth
- if (unordered.hasOwnProperty(code)
- && course_dict[code].depth === depth + 0.5) {
- for (const creq of course_dict[code].creqs) {
- if (course_dict.hasOwnProperty(creq)
- && course_dict[creq].depth === 0) {
- badcreq = true;
- }
- }
- if (!badcreq) {
- course_dict[code].depth = depth;
- }
- }
- }
- }
-
- this.setState({course_dict, min_height: 0});
- };
-
-
- // recursively check whether the given requirements are satisfied
-
- doneReqs(reqs) {
- if (reqs.length === 0) {
- return "done";
- }
-
- // recursively push status of each clause into status array
- const statuses = [];
- const operator = reqs[0];
- for (let i = 1; i < reqs.length; i++) {
- if (reqs[i] instanceof Array) {
- statuses.push(this.doneReqs(reqs[i]));
- }
- // push status of each course in the current tree
- else if (this.state.course_dict.hasOwnProperty(reqs[i])) {
- if (this.state.course_dict[reqs[i]].needs === "done") {
- statuses.push("done");
- } else {
- statuses.push("none");
- }
- }
- // courses outside the current tree are unknown
- else {
- statuses.push("outs");
- }
- }
-
- if (operator === "and") {
- if (statuses.indexOf("none") !== -1) {
- return "none"; // any course is none -> none
- } else if (statuses.indexOf("outs") !== -1) {
- return "outs"; // any course is outs -> outs
- } else {
- return "done"; // all courses are done -> done
- }
- } else if (operator === "or") {
- if (statuses.indexOf("done") !== -1) {
- return "done"; // any course is done -> done
- } else if (statuses.indexOf("outs") !== -1) {
- return "outs"; // any course is outs -> outs
- } else {
- return "none"; // all courses are none -> none
- }
- }
- };
-
-
- // update the status of the course with the given code
-
- updateCourse(code) {
- // courses can be taken in onClick -> done
- const button = this.state.course_dict[code];
- if (button.needs !== "done") {
- const course = course_data[code];
- // if any excluded course in the current tree is done -> excl
- if (course.excl.length > 1 && this.doneReqs(course.excl) === "done") {
- button.needs = "excl";
- // if any prerequisite in the current tree is not done -> preq
- } else if (this.doneReqs(course.preq) === "none") {
- button.needs = "preq";
- // if any corequisite in the current tree is not done -> creq
- } else if (this.doneReqs(course.creq) === "none") {
- button.needs = "creq";
- // if all prerequisites are in the current tree and done, and
- // all corequisites are in the current tree and done, then check...
- } else if (this.doneReqs(course.preq) === "done"
- && this.doneReqs(course.creq) === "done") {
- // if all excluded courses are in the current tree and not done -> none
- if (course.excl.length <= 1 || this.doneReqs(course.excl) === "none") {
- button.needs = "none";
- // otherwise, some excluded course is not in the current tree -> xout
- } else {
- button.needs = "xout";
- }
- // otherwise, some requisite course is not in the current tree -> outs
- } else {
- button.needs = "outs";
- }
- }
- };
-
-
- // callback to toggle the done status of a course
-
- updateNeeds(code) {
- const course_dict = {...this.state.course_dict};
- if (course_dict[code].needs === "done") {
- course_dict[code].needs = "none";
- } else {
- course_dict[code].needs = "done";
- }
- this.setState({course_dict});
- };
-
-
- // draw the main app
-
- render() {
- switch (this.props.tab) {
- case "helper":
- return (
- - Click on the tabs above to navigate between different tools. -
-- On the Course Browser, click on a course button to toggle whether you've taken that course already or not. The course colours are based on what courses you've already taken. -
-- Advanced users can use a Javascript-style regex in place of a course code when searching for courses to add. All course codes have no whitespace in them, and the regex is automatically enclosed with ^ and $. -
--
- {Object.entries(constants.button_colours).map(([needs, colour]) => {
- return (
-
- - A course with a{" "} - - {colour} button - - {constants.button_descriptions[needs]} - - ); - })} -
-
- {Object.entries(constants.border_colours).map(([needs, colour]) => {
- return (
-
- - A course with a{" "} - - {colour} border - - {constants.border_descriptions[needs]} - - ); - })} -
-
- );
- case "browser":
- return (
-
-
- );
- }
- };
-};
diff --git a/ubcexplorerscript.py b/ubcexplorerscript.py
index db7ba6f..adb0f57 100644
--- a/ubcexplorerscript.py
+++ b/ubcexplorerscript.py
@@ -1,224 +1,44 @@
+from typing import List
+from scraper import Scraper
import json
-import os
-import re
-from operator import itemgetter
-## pathnames to the .txt file and .json output directories. All output files will appear in /data/ubc/2020/json-output
-## this folder is in the .gitignore so they won't be staged for commit
-dirname = os.path.dirname(__file__)
-txt_file_directory = os.path.join(dirname, 'data/ubc/2020/courses')
-json_output_directory = os.path.join(dirname, 'data/ubc/2020/json-output')
-## global variables
-courses_array = []
-course = {}
-
-## Author: @MrBenC88
-## REQUIRES: ubcalend_txt.py to already have generated the .txt files
-## EFFECTS: parses .txt files generated by ubcalend_txt.py and converts into json objects
-## OUTPUT: output.json
-def generate_json_array():
- try:
- newfile = open(json_output_directory + '/' + "output.json", "w")
- except:
- os.mkdir(json_output_directory)
- newfile = open(json_output_directory + '/' + "output.json", "w")
-
- for file in os.listdir(txt_file_directory):
- f = open(txt_file_directory + '/' + str(file), "r")
-
- for line in f:
- if len(line) > 3:
- key, value = line.rstrip().split(': ', 1)
- course[key] = value.strip()
-
- else:
- if course:
- courses_array.append(course.copy())
- course.clear()
- continue
-
- newfile.write(json.dumps(courses_array))
- newfile.close()
-
-## this is used in credits_to_int() and dept_codes_to_json() - don't remove
-codes = []
-
-
-## REQUIRES: jsonOut directory to be present and ubcalend_txt.py to already have generated the .txt files
-## generate_json_array() must run before this function
-## EFFECTS: if single credit is specified in course["cred"], convert to int, otherwise set course["cred"] to null
-## OUTPUT: output.json (modified)
-def credits_to_int():
- f = open(json_output_directory + "/" + "output.json", "r")
- data = json.load(f)
- for course in data:
- if "cred" in course and len(course["cred"]) == 1:
- course["cred"] = int(course["cred"])
- else:
- course["cred"] = None
- codes.append(course["dept"])
- f.close()
- res = open(json_output_directory + "/" + "output.json", "w")
- res.write(json.dumps(data))
- res.close()
-
-
-## global variables for use in dept_codes_to_json
-codeSet = set()
-deptJson = {}
-deptArray = []
-
-## REQUIRES: jsonOut directory to be present and ubcalend_txt.py to already have generated the .txt files
-## generate_json_array() must run before this function
-## EFFECTS: creates a depts.json file with list of all deparments in alphabetical order (not for use with UBC Explorer)
-## OUTPUT: depts.json
-def dept_codes_to_json():
- for dept in codes:
- codeSet.add(dept)
-
- array = []
- for code in codeSet:
- array.append(code)
-
- array.sort()
-
- for dept in array:
- deptJson["dept"] = dept
- deptArray.append(deptJson.copy())
- deptJson.clear()
- newfile = open(json_output_directory + '/' + "depts.json", "w")
- newfile.write(json.dumps(deptArray))
-
-
-## REQUIRES: jsonOut directory to be present and ubcalend_txt.py to already have generated the .txt files
-## generate_json_array() must run before this function
-## EFFECTS: parses course["preq"] and course["creq"] fields in output.json and extracts strings with
-## 3+ capital letters and 3 consecutive digits into an array and replaces the course["preq"] and
-## course["creq"] field with respective arrays
-## OUTPUT: prereqs.json
-def parse_prereqs():
- f = open(json_output_directory + "/" + "output.json", "r")
- data = json.load(f)
-
- for course in data:
- if "preq" in course:
- course["preq"] = re.findall(r'[A-Z]*\s\d{3}[A-Z]*', course["preq"])
- if "creq" in course:
- course["creq"] = re.findall(r'[A-Z]*\s\d{3}[A-Z]*', course["creq"])
-
- f.close()
- res = open(json_output_directory + "/" + "prereqs.json", "w")
- res.write(json.dumps(data))
- res.close()
-
-
-## REQUIRES: jsonOut directory to be present and prereqs.json to have been generated
-## parse_prereqs() must run before this function
-## EFFECTS: for each course, looks for that course code in all of the courses' prerequisite arrays.
-## if match found, add to current course's course["depn"] array
-## OUTPUT: final_courses.json (not used with UBC Explorer)
-def get_dependencies():
- f = open(json_output_directory + "/" + "prereqs.json", "r")
- data = json.load(f)
-
- for course in data:
- for course2 in data:
- if "preq" in course2:
- if course["code"] in course2["preq"]:
- if not "depn" in course:
- course["depn"] = []
- course["depn"].append(course2["code"])
-
- f.close()
- res = open(json_output_directory + "/" + "final_courses.json", "w")
- res.write(json.dumps(data))
- res.close()
-
-
-## REQUIRES: jsonOut directory to be present and prereqs.json and depts.json to have been generated
-## parse_prereqs() must run before this function
-## dept_codes_to_json() must run before this function
-## EFFECTS: generates json array of all departments and their respective course codes
-## (not used with UBC Explorer)
-## OUTPUT: final_dept_codes.json
-def get_codes_for_depts():
- depts = open(json_output_directory + "/" + "depts.json", "r")
- courses = open(json_output_directory + "/" + "prereqs.json", "r")
-
- deptData = json.load(depts)
- courseData = json.load(courses)
-
- result = []
-
- for dept in deptData:
- newDept = {}
- newDept["dept"] = dept["dept"]
- newDept["courses"] = []
- for course in courseData:
- if course["dept"] == dept["dept"]:
- newDept["courses"].append(course["code"])
- result.append(newDept.copy())
-
- for dept in result:
- codesOnly = []
- for course in dept["courses"]:
- x = course.split(' ')
- codesOnly.append(x[1])
- dept["courses"] = codesOnly
-
- res = open(json_output_directory + "/" + "final_dept_codes.json", "w")
- res.write(json.dumps(result))
- res.close()
-
-
-## REQUIRES: jsonOut directory to be present and prereqs.json and depts.json to have been generated
-## parse_prereqs() must run before this function
-## dept_codes_to_json() must run before this function
-## EFFECTS: generates json array of all course codes (not used with UBC Explorer)
-## OUTPUT: course_codes.json
-def generate_course_codes():
- courses = open(json_output_directory + "/" + "prereqs.json", "r")
- data = json.load(courses)
- course_array = []
- for course in data:
- new_course = {}
- new_course["name"] = course["code"]
- course_array.append(new_course.copy())
- new_course.clear()
-
- course_array.sort(key=itemgetter("name"))
-
- res = open(json_output_directory + "/" + "course_codes.json", "w")
- res.write(json.dumps(course_array))
- res.close()
-
-
-## REQUIRES: jsonOut directory to be present and prereqs.json and depts.json to have been generated
-## parse_prereqs() must run before this function
-## dept_codes_to_json() must run before this function
-## EFFECTS: adds a field with all the UBC Courses URLs associated with each course
-## OUTPUT: with-links.json (this is the one that gets imported to UBC Explorer)
-def generage_ubc_courses_links():
- courses = open(json_output_directory + "/" + "final_courses.json", "r")
- data = json.load(courses)
- course_array = []
- for course in data:
- code = course["code"].split(" ")
- course["link"] = "https://courses.students.ubc.ca/cs/courseschedule?pname=subjarea&tname=subj-course&dept=" + code[0] + "&course=" + code[1]
- course_array.append(course.copy())
-
- res = open(json_output_directory + "/" + "with_links.json", "w")
- res.write(json.dumps(course_array))
- res.close()
-
-
-# call to functions (comment any out that you don't want to run)
-generate_json_array()
-credits_to_int()
-dept_codes_to_json()
-parse_prereqs()
-get_dependencies()
-get_codes_for_depts()
-generate_course_codes()
-generage_ubc_courses_links() ## this is the final file we use with UBC Explorer
\ No newline at end of file
+class UBCExplorerScript:
+ # for each course in the course array, search for all instances where the current course is a prereq for another course
+ # then adds courses found to the 'depn' array which represents the list of dependencies for that course
+ # returns the modified course array
+ def __get_dependencies(self, courses_array: List[dict]) -> List[dict]:
+ courses_map = {course["code"]: course for course in courses_array}
+ courses_with_preqs = [course for course in courses_array if "preq" in course]
+
+ for course in courses_with_preqs:
+ for preq in course["preq"]:
+ if preq in courses_map and preq != course["code"]:
+ to_update = courses_map[preq]
+ if not "depn" in to_update:
+ to_update["depn"] = set()
+ to_update["depn"].add(course["code"])
+
+ for course in courses_map.values():
+ if "depn" in course:
+ course["depn"] = sorted(list(course["depn"]))
+
+ return [value for value in courses_map.values()]
+
+ # generates final list of course objects to be uploaded to mongo
+ # returns final course array
+ def __generate_json_array(self, calendar_data: List[dict]) -> List[dict]:
+ sorted_courses = sorted(calendar_data, key=lambda course: course["code"])
+ return self.__get_dependencies(sorted_courses)
+
+ def main(self) -> List[dict]:
+ scraper = Scraper()
+ calendar_data = scraper.main()
+ return self.__generate_json_array(calendar_data)
+
+
+# testing
+# script = UBCExplorerScript()
+# output = script.main()
+# with open('output.json', 'w') as file:
+# json.dump(output, file)
diff --git a/webpack.config.js b/webpack.config.js
deleted file mode 100644
index bb27306..0000000
--- a/webpack.config.js
+++ /dev/null
@@ -1,25 +0,0 @@
-const path = require('path');
-const webpack = require('webpack');
-module.exports = {
- context: path.resolve(__dirname, './src'),
- entry: {
- app: './app.jsx',
- },
- output: {
- path: path.resolve(__dirname, './dist'),
- filename: 'req.js',
- publicPath: '/dist/',
- },
- module: {
- loaders: [
- {
- test: /.jsx?$/,
- loader: 'babel-loader',
- exclude: /node_modules/,
- query: {
- presets: ['es2015', 'react', 'stage-3']
- }
- }
- ]
- }
-};