From fc4febf76667e4936e8c99aa1209ae42491cceae Mon Sep 17 00:00:00 2001 From: jerryzhou196 Date: Wed, 1 Jul 2026 14:55:44 -0600 Subject: [PATCH 1/2] Add course planner backend: transcript grades in response, plan + checklist tables - Transcript parser now extracts per-course units and numeric grades and returns a per-term breakdown in the /parse/transcript response. Grades are returned for the client's local GPA calculator only and are never written to the database (per privacy policy). - New migration: user_course_plan (courses planned into future terms, modeled on user_shortlist) and checklist (maintainer-seeded degree requirement checklists as JSONB, each requirement a list of "one of" alternatives), with Hasura metadata and user permissions. Co-Authored-By: Claude Fable 5 --- flow/api/parse/parse.go | 8 +- flow/api/parse/transcript/transcript.go | 50 +++++- flow/api/parse/transcript/transcript_test.go | 166 +++++++++++++----- .../default/tables/public_checklist.yaml | 18 ++ .../tables/public_user_course_plan.yaml | 33 ++++ .../databases/default/tables/tables.yaml | 2 + .../1782938317000_add_course_planner/down.sql | 2 + .../1782938317000_add_course_planner/up.sql | 153 ++++++++++++++++ 8 files changed, 382 insertions(+), 50 deletions(-) create mode 100644 hasura/metadata/databases/default/tables/public_checklist.yaml create mode 100644 hasura/metadata/databases/default/tables/public_user_course_plan.yaml create mode 100644 hasura/migrations/default/1782938317000_add_course_planner/down.sql create mode 100644 hasura/migrations/default/1782938317000_add_course_planner/up.sql diff --git a/flow/api/parse/parse.go b/flow/api/parse/parse.go index b0d594b10..ce3a56966 100644 --- a/flow/api/parse/parse.go +++ b/flow/api/parse/parse.go @@ -17,6 +17,10 @@ import ( type transcriptResponse struct { CoursesImported int `json:"courses_imported"` + // Per-term course details (units, grades) parsed from the transcript. + // Returned to the client for its local GPA calculator; grades are + // deliberately never written to the database (see privacy policy). + Terms []transcript.TermSummary `json:"terms"` } const updateProgramQuery = ` @@ -59,11 +63,11 @@ func saveTranscript(tx *db.Tx, summary *transcript.Summary, userId int) (*transc return nil, fmt.Errorf("deleting old courses: %w", err) } - var response transcriptResponse + response := transcriptResponse{Terms: summary.TermSummaries} for _, termSummary := range summary.TermSummaries { response.CoursesImported += len(termSummary.Courses) for _, course := range termSummary.Courses { - _, err = tx.Exec(insertTranscriptQuery, course, userId, termSummary.TermId, termSummary.Level) + _, err = tx.Exec(insertTranscriptQuery, course.Code, userId, termSummary.TermId, termSummary.Level) if err != nil { return nil, fmt.Errorf("updating user_course_taken: %w", err) } diff --git a/flow/api/parse/transcript/transcript.go b/flow/api/parse/transcript/transcript.go index bf7f5c4f3..c5b399e64 100644 --- a/flow/api/parse/transcript/transcript.go +++ b/flow/api/parse/transcript/transcript.go @@ -9,13 +9,25 @@ import ( "flow/common/util" ) +type TermCourse struct { + // Course codes are similar to CS 145, STAT 920, PD 1, CHINA 120R, + // normalized to the form "cs145". + Code string `json:"code"` + // Units attempted (e.g. 0.5); 0 when the transcript does not list them + // (current-term courses). + Units float64 `json:"units"` + // Grade is the numeric grade out of 100; nil for non-numeric grades + // (CR, NCR, letter grades) and current-term courses. Grades are returned + // to the client for display only and are never persisted. + Grade *int `json:"grade"` +} + type TermSummary struct { // Term ids are numbers of the form 1189 (Fall 2018) - TermId int + TermId int `json:"term_id"` // Levels are similar to 1A, 5C (delayed graduation). - Level string - // Course codes are similar to CS 145, STAT 920, PD 1, CHINA 120R. - Courses []string + Level string `json:"level"` + Courses []TermCourse `json:"courses"` } type Summary struct { @@ -54,6 +66,26 @@ func isTransferCredit(courseLine string) bool { return len(matches) == 1 } +// extractUnitsAndGrade pulls the attempted units (first credit column) and the +// numeric grade (token after the last credit column) out of a course line. +// Past term course lines look like "ECON 102 Macroeconomics 0.50 0.50 98"; +// the grade may also be non-numeric (CR, NCR), in which case it is nil. +func extractUnitsAndGrade(courseLine string) (float64, *int) { + credits := creditRegexp.FindAllStringIndex(courseLine, -1) + if len(credits) < 2 { + return 0, nil + } + units, err := strconv.ParseFloat(courseLine[credits[0][0]:credits[0][1]], 64) + if err != nil { + units = 0 + } + rest := strings.TrimSpace(courseLine[credits[len(credits)-1][1]:]) + if grade, err := strconv.Atoi(rest); err == nil { + return units, &grade + } + return units, nil +} + func extractTermSummaries(text string) ([]TermSummary, error) { // Passing -1 means setting no upper limit on number of matches terms := termRegexp.FindAllStringSubmatchIndex(text, -1) @@ -75,16 +107,22 @@ func extractTermSummaries(text string) ([]TermSummary, error) { // Include courses that come before next term's heading // except for the last term, which includes all remaining courses. for ; j < len(courses) && (i == len(terms)-1 || courses[j][0] < terms[i+1][0]); j++ { + courseLine := text[courses[j][0]:courses[j][1]] // Some courses are transfer (AP/IB) credits. // They were not taken at UW, so should not be imported. - if isTransferCredit(text[courses[j][0]:courses[j][1]]) { + if isTransferCredit(courseLine) { continue } department := text[courses[j][2]:courses[j][3]] number := text[courses[j][4]:courses[j][5]] + units, grade := extractUnitsAndGrade(courseLine) history[i].Courses = append( history[i].Courses, - strings.ToLower(department+number), + TermCourse{ + Code: strings.ToLower(department + number), + Units: units, + Grade: grade, + }, ) } } diff --git a/flow/api/parse/transcript/transcript_test.go b/flow/api/parse/transcript/transcript_test.go index 503dc2553..f6656e2ba 100644 --- a/flow/api/parse/transcript/transcript_test.go +++ b/flow/api/parse/transcript/transcript_test.go @@ -10,6 +10,10 @@ import ( "github.com/google/go-cmp/cmp" ) +func grade(n int) *int { + return &n +} + func TestParseTranscript(t *testing.T) { tests := []struct { name string @@ -22,34 +26,68 @@ func TestParseTranscript(t *testing.T) { ProgramName: "Computer Science/Digital Hardware Option", TermSummaries: []TermSummary{ { - TermId: 1179, - Level: "1A", - Courses: []string{"cs145", "math145", "math147", "psych101", "spcom223"}, + TermId: 1179, + Level: "1A", + Courses: []TermCourse{ + {Code: "cs145", Units: 0.5, Grade: grade(97)}, + {Code: "math145", Units: 0.5, Grade: grade(95)}, + {Code: "math147", Units: 0.5, Grade: grade(97)}, + {Code: "psych101", Units: 0.5, Grade: grade(84)}, + {Code: "spcom223", Units: 0.5, Grade: grade(85)}, + }, }, { - TermId: 1181, - Level: "1B", - Courses: []string{"cs146", "ece124", "engl306a", "math146", "math148", "pd1", "stat230"}, + TermId: 1181, + Level: "1B", + Courses: []TermCourse{ + {Code: "cs146", Units: 0.5, Grade: grade(100)}, + {Code: "ece124", Units: 0.5, Grade: grade(95)}, + {Code: "engl306a", Units: 0.5, Grade: grade(93)}, + {Code: "math146", Units: 0.5, Grade: grade(100)}, + {Code: "math148", Units: 0.5, Grade: grade(92)}, + {Code: "pd1", Units: 0.5}, // CR + {Code: "stat230", Units: 0.5, Grade: grade(90)}, + }, }, { - TermId: 1185, - Level: "2A", - Courses: []string{"coop1", "pd11"}, + TermId: 1185, + Level: "2A", + Courses: []TermCourse{ + {Code: "coop1", Units: 0.5}, + {Code: "pd11", Units: 0.5}, + }, }, { - TermId: 1189, - Level: "2A", - Courses: []string{"cs241e", "cs245", "cs246e", "ece222", "math249"}, + TermId: 1189, + Level: "2A", + Courses: []TermCourse{ + {Code: "cs241e", Units: 0.5, Grade: grade(100)}, + {Code: "cs245", Units: 0.5, Grade: grade(91)}, + {Code: "cs246e", Units: 0.5, Grade: grade(100)}, + {Code: "ece222", Units: 0.5, Grade: grade(100)}, + {Code: "math249", Units: 0.5, Grade: grade(81)}, + }, }, { - TermId: 1191, - Level: "2B", - Courses: []string{"coop2", "pd10", "wkrpt200m"}, + TermId: 1191, + Level: "2B", + Courses: []TermCourse{ + {Code: "coop2", Units: 0.5}, + {Code: "pd10", Units: 0.5}, + {Code: "wkrpt200m", Units: 0.13}, // NG + }, }, { - TermId: 1195, - Level: "2B", - Courses: []string{"cs240e", "cs370", "math245", "math247", "stat231"}, + TermId: 1195, + Level: "2B", + // Current term at time of export: no units/grades listed. + Courses: []TermCourse{ + {Code: "cs240e"}, + {Code: "cs370"}, + {Code: "math245"}, + {Code: "math247"}, + {Code: "stat231"}, + }, }, }, }, @@ -61,44 +99,88 @@ func TestParseTranscript(t *testing.T) { ProgramName: "Computer Science", TermSummaries: []TermSummary{ { - TermId: 1179, - Level: "1A", - Courses: []string{"cs137", "ece105", "math115", "math117", "math135", "se101"}, + TermId: 1179, + Level: "1A", + Courses: []TermCourse{ + {Code: "cs137", Units: 0.5, Grade: grade(86)}, + {Code: "ece105", Units: 0.5, Grade: grade(75)}, + {Code: "math115", Units: 0.5, Grade: grade(90)}, + {Code: "math117", Units: 0.5, Grade: grade(93)}, + {Code: "math135", Units: 0.5, Grade: grade(87)}, + {Code: "se101", Units: 0.25, Grade: grade(98)}, + }, }, { - TermId: 1181, - Level: "1B", - Courses: []string{"cs138", "ece106", "ece124", "ece140", "math119"}, + TermId: 1181, + Level: "1B", + Courses: []TermCourse{ + {Code: "cs138", Units: 0.5, Grade: grade(89)}, + {Code: "ece106", Units: 0.5, Grade: grade(72)}, + {Code: "ece124", Units: 0.5, Grade: grade(84)}, + {Code: "ece140", Units: 0.5, Grade: grade(75)}, + {Code: "math119", Units: 0.5, Grade: grade(87)}, + }, }, { - TermId: 1185, - Level: "1B", - Courses: []string{"coop1", "pd20"}, + TermId: 1185, + Level: "1B", + Courses: []TermCourse{ + {Code: "coop1", Units: 0.5}, + {Code: "pd20", Units: 0.5}, + }, }, { - TermId: 1189, - Level: "2A", - Courses: []string{"che102", "cs241e", "ece222", "se212", "smf213", "spcom223", "stat206"}, + TermId: 1189, + Level: "2A", + Courses: []TermCourse{ + {Code: "che102", Units: 0.5, Grade: grade(84)}, + {Code: "cs241e", Units: 0.5, Grade: grade(78)}, + {Code: "ece222", Units: 0.5, Grade: grade(89)}, + {Code: "se212", Units: 0.5, Grade: grade(73)}, + {Code: "smf213", Units: 0.5, Grade: grade(83)}, + {Code: "spcom223", Units: 0.5, Grade: grade(85)}, + {Code: "stat206", Units: 0.5, Grade: grade(86)}, + }, }, { - TermId: 1191, - Level: "2A", - Courses: []string{"coop2", "pd21"}, + TermId: 1191, + Level: "2A", + Courses: []TermCourse{ + {Code: "coop2", Units: 0.5}, + {Code: "pd21", Units: 0.5}, + }, }, { - TermId: 1195, - Level: "2B", - Courses: []string{"cs240", "cs247", "earth121", "ece358", "math239", "msci261", "wkrpt200"}, + TermId: 1195, + Level: "2B", + Courses: []TermCourse{ + {Code: "cs240", Units: 0.5, Grade: grade(85)}, + {Code: "cs247", Units: 0.5, Grade: grade(89)}, + {Code: "earth121", Units: 0.5, Grade: grade(83)}, + {Code: "ece358", Units: 0.5, Grade: grade(80)}, + {Code: "math239", Units: 0.5, Grade: grade(74)}, + {Code: "msci261", Units: 0.5, Grade: grade(87)}, + {Code: "wkrpt200", Units: 0.13, Grade: grade(95)}, + }, }, { - TermId: 1199, - Level: "2B", - Courses: []string{"coop3", "pd10"}, + TermId: 1199, + Level: "2B", + Courses: []TermCourse{ + {Code: "coop3", Units: 0.5}, + {Code: "pd10", Units: 0.5}, + }, }, { - TermId: 1201, - Level: "4A", - Courses: []string{"cs341", "cs350", "cs370", "phil256", "syde552"}, + TermId: 1201, + Level: "4A", + Courses: []TermCourse{ + {Code: "cs341"}, + {Code: "cs350"}, + {Code: "cs370"}, + {Code: "phil256"}, + {Code: "syde552"}, + }, }, }, }, diff --git a/hasura/metadata/databases/default/tables/public_checklist.yaml b/hasura/metadata/databases/default/tables/public_checklist.yaml new file mode 100644 index 000000000..e6ff4f983 --- /dev/null +++ b/hasura/metadata/databases/default/tables/public_checklist.yaml @@ -0,0 +1,18 @@ +table: + name: checklist + schema: public +select_permissions: + - role: anonymous + permission: + columns: + - id + - name + - requirements + filter: {} + - role: user + permission: + columns: + - id + - name + - requirements + filter: {} diff --git a/hasura/metadata/databases/default/tables/public_user_course_plan.yaml b/hasura/metadata/databases/default/tables/public_user_course_plan.yaml new file mode 100644 index 000000000..d2ee61a1a --- /dev/null +++ b/hasura/metadata/databases/default/tables/public_user_course_plan.yaml @@ -0,0 +1,33 @@ +table: + name: user_course_plan + schema: public +object_relationships: + - name: course + using: + foreign_key_constraint_on: course_id +insert_permissions: + - role: user + permission: + check: + user_id: + _eq: X-Hasura-User-Id + columns: + - course_id + - user_id + - term_id +select_permissions: + - role: user + permission: + columns: + - course_id + - user_id + - term_id + filter: + user_id: + _eq: X-Hasura-User-Id +delete_permissions: + - role: user + permission: + filter: + user_id: + _eq: X-Hasura-User-Id diff --git a/hasura/metadata/databases/default/tables/tables.yaml b/hasura/metadata/databases/default/tables/tables.yaml index 43d191be5..210ab88cc 100644 --- a/hasura/metadata/databases/default/tables/tables.yaml +++ b/hasura/metadata/databases/default/tables/tables.yaml @@ -6,6 +6,7 @@ - "!include aggregate_prof_engaging_buckets.yaml" - "!include aggregate_prof_rating.yaml" - "!include aggregate_prof_review_rating.yaml" +- "!include public_checklist.yaml" - "!include public_course.yaml" - "!include public_course_antirequisite.yaml" - "!include public_course_postrequisite.yaml" @@ -23,6 +24,7 @@ - "!include public_section_exam.yaml" - "!include public_section_meeting.yaml" - "!include public_user.yaml" +- "!include public_user_course_plan.yaml" - "!include public_user_course_taken.yaml" - "!include public_user_schedule.yaml" - "!include public_user_shortlist.yaml" diff --git a/hasura/migrations/default/1782938317000_add_course_planner/down.sql b/hasura/migrations/default/1782938317000_add_course_planner/down.sql new file mode 100644 index 000000000..edfb0c64a --- /dev/null +++ b/hasura/migrations/default/1782938317000_add_course_planner/down.sql @@ -0,0 +1,2 @@ +DROP TABLE user_course_plan; +DROP TABLE checklist; diff --git a/hasura/migrations/default/1782938317000_add_course_planner/up.sql b/hasura/migrations/default/1782938317000_add_course_planner/up.sql new file mode 100644 index 000000000..d65148ed7 --- /dev/null +++ b/hasura/migrations/default/1782938317000_add_course_planner/up.sql @@ -0,0 +1,153 @@ +-- Courses a user plans to take in a future term (the /plan page). +-- Mirrors user_course_taken, but is user-editable via Hasura. +CREATE TABLE user_course_plan ( + user_id INT NOT NULL + REFERENCES "user"(id) + ON DELETE CASCADE + ON UPDATE CASCADE, + course_id INT NOT NULL + REFERENCES course(id) + ON DELETE CASCADE + ON UPDATE CASCADE, + term_id INT NOT NULL, + PRIMARY KEY(user_id, term_id, course_id) +); + +CREATE INDEX user_course_plan_user_id_fkey ON user_course_plan(user_id); + +-- Degree requirement checklists maintained by UW Flow (seeded below, updated +-- by maintainers via SQL). `requirements` is an ordered array of categories: +-- [{"category": "Computer science", "courses": [["cs135","cs115"], ...]}] +-- Each entry of `courses` is a list of alternatives ("one of"). +-- ponytail: flat one-of lists only; "n additional courses from X" style +-- requirements need a richer schema if we ever want them. +CREATE TABLE checklist ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + requirements JSONB NOT NULL +); + +INSERT INTO checklist(name, requirements) VALUES +( + 'Honours Computer Science (core)', + '[ + { + "category": "Computer science", + "courses": [ + ["cs135", "cs115", "cs145"], + ["cs136", "cs146"], + ["cs136l"], + ["cs240", "cs240e"], + ["cs241", "cs241e"], + ["cs245", "cs245e"], + ["cs246", "cs246e"], + ["cs251", "cs251e"], + ["cs341"], + ["cs350"] + ] + }, + { + "category": "Mathematics", + "courses": [ + ["math135", "math145"], + ["math136", "math146"], + ["math137", "math147"], + ["math138", "math148"], + ["math239", "math249"], + ["stat230", "stat240"], + ["stat231", "stat241"] + ] + }, + { + "category": "Communication", + "courses": [ + ["engl109", "engl119", "spcom100", "spcom223", "emls101r", "emls102r", "engl129r"], + ["engl209", "engl210e", "engl210f", "spcom225", "spcom227", "spcom228", "emls103r"] + ] + } + ]'::jsonb +), +( + 'Honours Mathematics (core)', + '[ + { + "category": "Mathematics core", + "courses": [ + ["math135", "math145"], + ["math136", "math146"], + ["math137", "math147"], + ["math138", "math148"], + ["math235", "math245"], + ["math237", "math247", "math239", "math249"], + ["stat230", "stat240"], + ["stat231", "stat241"] + ] + }, + { + "category": "Computer science", + "courses": [ + ["cs115", "cs135", "cs145"], + ["cs116", "cs136", "cs146"] + ] + }, + { + "category": "Communication", + "courses": [ + ["engl109", "engl119", "spcom100", "spcom223", "emls101r", "emls102r", "engl129r"] + ] + } + ]'::jsonb +), +( + 'Software Engineering (core)', + '[ + { + "category": "Software engineering", + "courses": [ + ["se101"], + ["se212"], + ["se350"], + ["se463"], + ["se464", "cs446", "ece452"], + ["se465", "cs447", "ece453"] + ] + }, + { + "category": "Computer science", + "courses": [ + ["cs137"], + ["cs138"], + ["cs240", "cs240e"], + ["cs241", "cs241e"], + ["cs247"], + ["cs341"], + ["cs343"], + ["cs348"], + ["cs349"] + ] + }, + { + "category": "Mathematics", + "courses": [ + ["math115"], + ["math117"], + ["math119"], + ["math135", "math145"], + ["math239", "math249"], + ["stat206"] + ] + }, + { + "category": "Engineering", + "courses": [ + ["ece105"], + ["ece106"], + ["ece124"], + ["ece140"], + ["ece222"], + ["ece358"], + ["msci261"] + ] + } + ]'::jsonb +); From 789bb7cb3d037103ccc268f14820f9474f427900 Mon Sep 17 00:00:00 2001 From: jerryzhou196 Date: Wed, 1 Jul 2026 15:10:11 -0600 Subject: [PATCH 2/2] Batch transcript course inserts into a single statement user_course_taken has a per-statement trigger in prod that refreshes materialized.course_recommendation. Inserting one course at a time turned a transcript upload into ~40 sequential matview refreshes, easily exceeding the 10s request deadline ("deleting old courses: timeout"). One unnest-join INSERT brings the whole save down to two statements. courses_imported now reports rows actually inserted (unknown course codes no longer count). Co-Authored-By: Claude Fable 5 --- flow/api/parse/parse.go | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/flow/api/parse/parse.go b/flow/api/parse/parse.go index ce3a56966..3d11553ba 100644 --- a/flow/api/parse/parse.go +++ b/flow/api/parse/parse.go @@ -32,9 +32,15 @@ DELETE FROM user_course_taken WHERE term_id <= $1 AND user_id = $2 ` +// One INSERT for the whole transcript: user_course_taken carries a +// per-statement trigger in prod that refreshes a materialized view, so +// per-course INSERTs turn one upload into dozens of refreshes and blow the +// request deadline. const insertTranscriptQuery = ` INSERT INTO user_course_taken(course_id, user_id, term_id, level) -SELECT id, $2, $3, $4 FROM course WHERE code = $1 +SELECT course.id, $1, input.term_id, input.level +FROM unnest($2::text[], $3::int[], $4::text[]) AS input(code, term_id, level) +JOIN course ON course.code = input.code ` func saveTranscript(tx *db.Tx, summary *transcript.Summary, userId int) (*transcriptResponse, error) { @@ -63,18 +69,26 @@ func saveTranscript(tx *db.Tx, summary *transcript.Summary, userId int) (*transc return nil, fmt.Errorf("deleting old courses: %w", err) } - response := transcriptResponse{Terms: summary.TermSummaries} + var codes []string + var termIds []int32 + var levels []string for _, termSummary := range summary.TermSummaries { - response.CoursesImported += len(termSummary.Courses) for _, course := range termSummary.Courses { - _, err = tx.Exec(insertTranscriptQuery, course.Code, userId, termSummary.TermId, termSummary.Level) - if err != nil { - return nil, fmt.Errorf("updating user_course_taken: %w", err) - } + codes = append(codes, course.Code) + termIds = append(termIds, int32(termSummary.TermId)) + levels = append(levels, termSummary.Level) } } - return &response, nil + tag, err := tx.Exec(insertTranscriptQuery, userId, codes, termIds, levels) + if err != nil { + return nil, fmt.Errorf("updating user_course_taken: %w", err) + } + + return &transcriptResponse{ + CoursesImported: int(tag.RowsAffected()), + Terms: summary.TermSummaries, + }, nil } func HandleTranscript(tx *db.Tx, r *http.Request) (interface{}, error) {