Course planner backend: transcript grades in parse response, plan + checklist tables

flow/api/parse/parse.go

@@ -17,6 +17,10 @@ import (
 
 type transcriptResponse struct {
 	CoursesImported int `json:"courses_imported"`
+	// Per-term course details (units, grades) parsed from the transcript.
+	// Returned to the client for its local GPA calculator; grades are
+	// deliberately never written to the database (see privacy policy).
+	Terms []transcript.TermSummary `json:"terms"`
 }
 
 const updateProgramQuery = `
@@ -28,9 +32,15 @@ DELETE FROM user_course_taken
 WHERE term_id <= $1 AND user_id = $2
 `
 
+// One INSERT for the whole transcript: user_course_taken carries a
+// per-statement trigger in prod that refreshes a materialized view, so
+// per-course INSERTs turn one upload into dozens of refreshes and blow the
+// request deadline.
 const insertTranscriptQuery = `
 INSERT INTO user_course_taken(course_id, user_id, term_id, level)
-SELECT id, $2, $3, $4 FROM course WHERE code = $1
+SELECT course.id, $1, input.term_id, input.level
+FROM unnest($2::text[], $3::int[], $4::text[]) AS input(code, term_id, level)
+JOIN course ON course.code = input.code
 `
 
 func saveTranscript(tx *db.Tx, summary *transcript.Summary, userId int) (*transcriptResponse, error) {
@@ -59,18 +69,26 @@ func saveTranscript(tx *db.Tx, summary *transcript.Summary, userId int) (*transc
 		return nil, fmt.Errorf("deleting old courses: %w", err)
 	}
 
-	var response transcriptResponse
+	var codes []string
+	var termIds []int32
+	var levels []string
 	for _, termSummary := range summary.TermSummaries {
-		response.CoursesImported += len(termSummary.Courses)
 		for _, course := range termSummary.Courses {
-			_, err = tx.Exec(insertTranscriptQuery, course, userId, termSummary.TermId, termSummary.Level)
-			if err != nil {
-				return nil, fmt.Errorf("updating user_course_taken: %w", err)
-			}
+			codes = append(codes, course.Code)
+			termIds = append(termIds, int32(termSummary.TermId))
+			levels = append(levels, termSummary.Level)
 		}
 	}
 
-	return &response, nil
+	tag, err := tx.Exec(insertTranscriptQuery, userId, codes, termIds, levels)
+	if err != nil {
+		return nil, fmt.Errorf("updating user_course_taken: %w", err)
+	}
+
+	return &transcriptResponse{
+		CoursesImported: int(tag.RowsAffected()),
+		Terms:           summary.TermSummaries,
+	}, nil
 }
 
 func HandleTranscript(tx *db.Tx, r *http.Request) (interface{}, error) {

flow/api/parse/transcript/transcript.go

@@ -9,13 +9,25 @@ import (
 	"flow/common/util"
 )
 
+type TermCourse struct {
+	// Course codes are similar to CS 145, STAT 920, PD 1, CHINA 120R,
+	// normalized to the form "cs145".
+	Code string `json:"code"`
+	// Units attempted (e.g. 0.5); 0 when the transcript does not list them
+	// (current-term courses).
+	Units float64 `json:"units"`
+	// Grade is the numeric grade out of 100; nil for non-numeric grades
+	// (CR, NCR, letter grades) and current-term courses. Grades are returned
+	// to the client for display only and are never persisted.
+	Grade *int `json:"grade"`
+}
+
 type TermSummary struct {
 	// Term ids are numbers of the form 1189 (Fall 2018)
-	TermId int
+	TermId int `json:"term_id"`
 	// Levels are similar to 1A, 5C (delayed graduation).
-	Level string
-	// Course codes are similar to CS 145, STAT 920, PD 1, CHINA 120R.
-	Courses []string
+	Level   string       `json:"level"`
+	Courses []TermCourse `json:"courses"`
 }
 
 type Summary struct {
@@ -54,6 +66,26 @@ func isTransferCredit(courseLine string) bool {
 	return len(matches) == 1
 }
 
+// extractUnitsAndGrade pulls the attempted units (first credit column) and the
+// numeric grade (token after the last credit column) out of a course line.
+// Past term course lines look like "ECON  102  Macroeconomics  0.50  0.50  98";
+// the grade may also be non-numeric (CR, NCR), in which case it is nil.
+func extractUnitsAndGrade(courseLine string) (float64, *int) {
+	credits := creditRegexp.FindAllStringIndex(courseLine, -1)
+	if len(credits) < 2 {
+		return 0, nil
+	}
+	units, err := strconv.ParseFloat(courseLine[credits[0][0]:credits[0][1]], 64)
+	if err != nil {
+		units = 0
+	}
+	rest := strings.TrimSpace(courseLine[credits[len(credits)-1][1]:])
+	if grade, err := strconv.Atoi(rest); err == nil {
+		return units, &grade
+	}
+	return units, nil
+}
+
 func extractTermSummaries(text string) ([]TermSummary, error) {
 	// Passing -1 means setting no upper limit on number of matches
 	terms := termRegexp.FindAllStringSubmatchIndex(text, -1)
@@ -75,16 +107,22 @@ func extractTermSummaries(text string) ([]TermSummary, error) {
 		// Include courses that come before next term's heading
 		// except for the last term, which includes all remaining courses.
 		for ; j < len(courses) && (i == len(terms)-1 || courses[j][0] < terms[i+1][0]); j++ {
+			courseLine := text[courses[j][0]:courses[j][1]]
 			// Some courses are transfer (AP/IB) credits.
 			// They were not taken at UW, so should not be imported.
-			if isTransferCredit(text[courses[j][0]:courses[j][1]]) {
+			if isTransferCredit(courseLine) {
 				continue
 			}
 			department := text[courses[j][2]:courses[j][3]]
 			number := text[courses[j][4]:courses[j][5]]
+			units, grade := extractUnitsAndGrade(courseLine)
 			history[i].Courses = append(
 				history[i].Courses,
-				strings.ToLower(department+number),
+				TermCourse{
+					Code:  strings.ToLower(department + number),
+					Units: units,
+					Grade: grade,
+				},
 			)
 		}
 	}

flow/api/parse/transcript/transcript_test.go

@@ -10,6 +10,10 @@ import (
 	"github.com/google/go-cmp/cmp"
 )
 
+func grade(n int) *int {
+	return &n
+}
+
 func TestParseTranscript(t *testing.T) {
 	tests := []struct {
 		name string
@@ -22,34 +26,68 @@ func TestParseTranscript(t *testing.T) {
 				ProgramName:   "Computer Science/Digital Hardware Option",
 				TermSummaries: []TermSummary{
 					{
-						TermId:  1179,
-						Level:   "1A",
-						Courses: []string{"cs145", "math145", "math147", "psych101", "spcom223"},
+						TermId: 1179,
+						Level:  "1A",
+						Courses: []TermCourse{
+							{Code: "cs145", Units: 0.5, Grade: grade(97)},
+							{Code: "math145", Units: 0.5, Grade: grade(95)},
+							{Code: "math147", Units: 0.5, Grade: grade(97)},
+							{Code: "psych101", Units: 0.5, Grade: grade(84)},
+							{Code: "spcom223", Units: 0.5, Grade: grade(85)},
+						},
 					},
 					{
-						TermId:  1181,
-						Level:   "1B",
-						Courses: []string{"cs146", "ece124", "engl306a", "math146", "math148", "pd1", "stat230"},
+						TermId: 1181,
+						Level:  "1B",
+						Courses: []TermCourse{
+							{Code: "cs146", Units: 0.5, Grade: grade(100)},
+							{Code: "ece124", Units: 0.5, Grade: grade(95)},
+							{Code: "engl306a", Units: 0.5, Grade: grade(93)},
+							{Code: "math146", Units: 0.5, Grade: grade(100)},
+							{Code: "math148", Units: 0.5, Grade: grade(92)},
+							{Code: "pd1", Units: 0.5}, // CR
+							{Code: "stat230", Units: 0.5, Grade: grade(90)},
+						},
 					},
 					{
-						TermId:  1185,
-						Level:   "2A",
-						Courses: []string{"coop1", "pd11"},
+						TermId: 1185,
+						Level:  "2A",
+						Courses: []TermCourse{
+							{Code: "coop1", Units: 0.5},
+							{Code: "pd11", Units: 0.5},
+						},
 					},
 					{
-						TermId:  1189,
-						Level:   "2A",
-						Courses: []string{"cs241e", "cs245", "cs246e", "ece222", "math249"},
+						TermId: 1189,
+						Level:  "2A",
+						Courses: []TermCourse{
+							{Code: "cs241e", Units: 0.5, Grade: grade(100)},
+							{Code: "cs245", Units: 0.5, Grade: grade(91)},
+							{Code: "cs246e", Units: 0.5, Grade: grade(100)},
+							{Code: "ece222", Units: 0.5, Grade: grade(100)},
+							{Code: "math249", Units: 0.5, Grade: grade(81)},
+						},
 					},
 					{
-						TermId:  1191,
-						Level:   "2B",
-						Courses: []string{"coop2", "pd10", "wkrpt200m"},
+						TermId: 1191,
+						Level:  "2B",
+						Courses: []TermCourse{
+							{Code: "coop2", Units: 0.5},
+							{Code: "pd10", Units: 0.5},
+							{Code: "wkrpt200m", Units: 0.13}, // NG
+						},
 					},
 					{
-						TermId:  1195,
-						Level:   "2B",
-						Courses: []string{"cs240e", "cs370", "math245", "math247", "stat231"},
+						TermId: 1195,
+						Level:  "2B",
+						// Current term at time of export: no units/grades listed.
+						Courses: []TermCourse{
+							{Code: "cs240e"},
+							{Code: "cs370"},
+							{Code: "math245"},
+							{Code: "math247"},
+							{Code: "stat231"},
+						},
 					},
 				},
 			},
@@ -61,44 +99,88 @@ func TestParseTranscript(t *testing.T) {
 				ProgramName:   "Computer Science",
 				TermSummaries: []TermSummary{
 					{
-						TermId:  1179,
-						Level:   "1A",
-						Courses: []string{"cs137", "ece105", "math115", "math117", "math135", "se101"},
+						TermId: 1179,
+						Level:  "1A",
+						Courses: []TermCourse{
+							{Code: "cs137", Units: 0.5, Grade: grade(86)},
+							{Code: "ece105", Units: 0.5, Grade: grade(75)},
+							{Code: "math115", Units: 0.5, Grade: grade(90)},
+							{Code: "math117", Units: 0.5, Grade: grade(93)},
+							{Code: "math135", Units: 0.5, Grade: grade(87)},
+							{Code: "se101", Units: 0.25, Grade: grade(98)},
+						},
 					},
 					{
-						TermId:  1181,
-						Level:   "1B",
-						Courses: []string{"cs138", "ece106", "ece124", "ece140", "math119"},
+						TermId: 1181,
+						Level:  "1B",
+						Courses: []TermCourse{
+							{Code: "cs138", Units: 0.5, Grade: grade(89)},
+							{Code: "ece106", Units: 0.5, Grade: grade(72)},
+							{Code: "ece124", Units: 0.5, Grade: grade(84)},
+							{Code: "ece140", Units: 0.5, Grade: grade(75)},
+							{Code: "math119", Units: 0.5, Grade: grade(87)},
+						},
 					},
 					{
-						TermId:  1185,
-						Level:   "1B",
-						Courses: []string{"coop1", "pd20"},
+						TermId: 1185,
+						Level:  "1B",
+						Courses: []TermCourse{
+							{Code: "coop1", Units: 0.5},
+							{Code: "pd20", Units: 0.5},
+						},
 					},
 					{
-						TermId:  1189,
-						Level:   "2A",
-						Courses: []string{"che102", "cs241e", "ece222", "se212", "smf213", "spcom223", "stat206"},
+						TermId: 1189,
+						Level:  "2A",
+						Courses: []TermCourse{
+							{Code: "che102", Units: 0.5, Grade: grade(84)},
+							{Code: "cs241e", Units: 0.5, Grade: grade(78)},
+							{Code: "ece222", Units: 0.5, Grade: grade(89)},
+							{Code: "se212", Units: 0.5, Grade: grade(73)},
+							{Code: "smf213", Units: 0.5, Grade: grade(83)},
+							{Code: "spcom223", Units: 0.5, Grade: grade(85)},
+							{Code: "stat206", Units: 0.5, Grade: grade(86)},
+						},
 					},
 					{
-						TermId:  1191,
-						Level:   "2A",
-						Courses: []string{"coop2", "pd21"},
+						TermId: 1191,
+						Level:  "2A",
+						Courses: []TermCourse{
+							{Code: "coop2", Units: 0.5},
+							{Code: "pd21", Units: 0.5},
+						},
 					},
 					{
-						TermId:  1195,
-						Level:   "2B",
-						Courses: []string{"cs240", "cs247", "earth121", "ece358", "math239", "msci261", "wkrpt200"},
+						TermId: 1195,
+						Level:  "2B",
+						Courses: []TermCourse{
+							{Code: "cs240", Units: 0.5, Grade: grade(85)},
+							{Code: "cs247", Units: 0.5, Grade: grade(89)},
+							{Code: "earth121", Units: 0.5, Grade: grade(83)},
+							{Code: "ece358", Units: 0.5, Grade: grade(80)},
+							{Code: "math239", Units: 0.5, Grade: grade(74)},
+							{Code: "msci261", Units: 0.5, Grade: grade(87)},
+							{Code: "wkrpt200", Units: 0.13, Grade: grade(95)},
+						},
 					},
 					{
-						TermId:  1199,
-						Level:   "2B",
-						Courses: []string{"coop3", "pd10"},
+						TermId: 1199,
+						Level:  "2B",
+						Courses: []TermCourse{
+							{Code: "coop3", Units: 0.5},
+							{Code: "pd10", Units: 0.5},
+						},
 					},
 					{
-						TermId:  1201,
-						Level:   "4A",
-						Courses: []string{"cs341", "cs350", "cs370", "phil256", "syde552"},
+						TermId: 1201,
+						Level:  "4A",
+						Courses: []TermCourse{
+							{Code: "cs341"},
+							{Code: "cs350"},
+							{Code: "cs370"},
+							{Code: "phil256"},
+							{Code: "syde552"},
+						},
 					},
 				},
 			},

hasura/metadata/databases/default/tables/public_checklist.yaml

@@ -0,0 +1,18 @@
+table:
+  name: checklist
+  schema: public
+select_permissions:
+  - role: anonymous
+    permission:
+      columns:
+        - id
+        - name
+        - requirements
+      filter: {}
+  - role: user
+    permission:
+      columns:
+        - id
+        - name
+        - requirements
+      filter: {}