From bb27d9157d924e9f45ffe6c6e7df74aeb435a733 Mon Sep 17 00:00:00 2001 From: Brendan-Lim Date: Wed, 11 Nov 2026 17:34:51 +0800 Subject: [PATCH 1/2] Add verified chain-rule practice and offline tutoring episode evaluation Offer bounded sine, cosine and exponential composition questions through the existing verified-key practice flow, recap recommendations and My Mistakes. Preserve verifier, uncertainty, accounting and mastery rules. Reuse existing HTTP doubles for ten offline tutoring and guard episodes, with comparable fixture/method hashes, regression reporting, independent symbolic key checks and a browser journey. Repair the existing adaptive report's dependence on catalog ordering. Document scope and limitations. --- apps/web/lib/types.ts | 2 +- apps/web/tests/e2e/learning.spec.ts | 39 +++ docs/ARCHITECTURE.md | 7 + docs/TESTING.md | 26 ++ evals/tutoring.py | 260 ++++++++++++++++++ scripts/public-files.txt | 3 + services/api/quack/schemas.py | 2 +- services/api/quack/services/practice.py | 21 ++ services/api/quack/services/topics.py | 2 +- services/api/scripts/adaptive_eval.py | 7 +- .../api/tests/test_chain_rule_practice.py | 108 ++++++++ services/api/tests/test_practice.py | 6 +- services/api/tests/test_tutoring_eval.py | 50 ++++ 13 files changed, 526 insertions(+), 7 deletions(-) create mode 100644 evals/tutoring.py create mode 100644 services/api/tests/test_chain_rule_practice.py create mode 100644 services/api/tests/test_tutoring_eval.py diff --git a/apps/web/lib/types.ts b/apps/web/lib/types.ts index 7b12590..c2c7f61 100644 --- a/apps/web/lib/types.ts +++ b/apps/web/lib/types.ts @@ -250,7 +250,7 @@ export type SessionHistory = { sessions: SessionHistoryItem[] }; // Practice. Nothing but PracticeReveal carries the answer. -export type PracticeTopic = "addition_subtraction" | "multiplication" | "division" | "expanding_brackets" | "combining_like_terms" | "solving_equations" | "power_rule" | "sum_and_constant_rules" | "integration"; +export type PracticeTopic = "addition_subtraction" | "multiplication" | "division" | "expanding_brackets" | "combining_like_terms" | "solving_equations" | "power_rule" | "sum_and_constant_rules" | "integration" | "chain_rule"; export type PracticeOutcome = "in_progress" | "incorrect" | "correct"; export type PracticeProblem = { diff --git a/apps/web/tests/e2e/learning.spec.ts b/apps/web/tests/e2e/learning.spec.ts index fd6d8f3..124f670 100644 --- a/apps/web/tests/e2e/learning.spec.ts +++ b/apps/web/tests/e2e/learning.spec.ts @@ -50,6 +50,45 @@ test.beforeEach(async () => { }); test.describe("the journeys", () => { + test("chain-rule correction leads to verified follow-up practice", async ({ page }) => { + test.setTimeout(180_000); + await signUp(page); + await script({ reading: reading(first("d/dx(sin(x^2)) = cos(x^2)")) }); + await write(page, "d/dx(sin(x^2)) = cos(x^2)", 80, 80, 24); + await expectDuck(page, "alert"); + await duck(page).click(); + await expect(page.getByTestId("hint-panel")).toContainText("function inside another"); + await expect(page.getByTestId("hint-panel")).not.toContainText("2x"); + await script({ reading: reading(first("d/dx(sin(x^2)) = cos(x^2)"), first("d/dx(sin(x^2)) = 2x*cos(x^2)")) }); + await write(page, "d/dx(sin(x^2)) = 2x*cos(x^2)", 80, 160, 24); + await expect(bubble(page)).toHaveText("That's it."); + await settle(page, 500); + await openMenu(page, "menu-mistakes"); + await entries(page).click(); + await expect(page.getByTestId("mistake-practise")).toBeVisible(); + await page.getByTestId("mistakes-close").click(); + await page.getByTestId("finish-session").click(); + await expect(page.getByTestId("recommendation")).toContainText("Chain rule"); + await page.getByTestId("practice-this").click(); + const prompt = page.getByTestId("practice-prompt"); + await expect(prompt).toContainText("d/dx("); + const match = (await prompt.innerText()).match(/d\/dx\((sin|cos|exp)\((\d+)x\)\)/); + expect(match).not.toBeNull(); + const [, family, scale] = match!; + const outer = family === "sin" ? `cos(${scale}x)` : family === "cos" ? `-sin(${scale}x)` : `exp(${scale}x)`; + const answer = `${scale}*(${outer})`; + await script({ reading: reading(first("0")) }); + await write(page, "0", 80, 80); + await expect(page.getByTestId("practice").getByTestId("bubble")).toContainText("Quack!"); + await script({ reading: reading(first(answer)) }); + await page.getByTestId("practice-clear").click(); + await write(page, answer, 80, 80, 24); + await expect(page.getByTestId("practice-correct")).toBeVisible(); + await page.getByTestId("practice-end").click(); + await expect(page.getByTestId("summary-correct")).toHaveText("1"); + await expect(page.getByTestId("summary-message")).toContainText("One question doesn't say much"); + }); + test("A: simplifying, a hint, a correction, My Mistakes, a recap, and practice", async ({ page }) => { test.setTimeout(180_000); await signUp(page); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ae5863f..70a4982 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -55,6 +55,13 @@ Generated answer keys are verified before questions are shown. `learning.py` val Independent completion earns one credit and assisted completion earns half. Distinct question types and skills contribute to mastery. Saved progress, My Mistakes and recaps support returning learners. +Standalone practice also supports chain-rule follow-ups from verified notebook mistakes. Explicit +integer-derived keys for sine, cosine and exponential compositions pass the existing bounded +verification/completion gate before display. Basic questions have a linear inner function; +intermediate ones have a power. This adds no model call during question generation and no new +mastery rule. A missing-factor pattern supports a useful hint without establishing the student's +mental cause. The offline episode scorecard in `evals/tutoring.py` exercises this flow and its guards. + ## Accounts and storage Notes remain in the browser's IndexedDB, keyed by account. Calibration samples, study sessions, mistake facts, learning events and progression are stored server-side, using SQLite locally or PostgreSQL through Psycopg. Notebook notes do not synchronise across devices; account learning records can. diff --git a/docs/TESTING.md b/docs/TESTING.md index 180e4c6..d0545be 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -43,6 +43,32 @@ This starts `next start` and verifies that `/collect` returns HTTP 404, `?debug` Synthetic fixtures under `evals/synthetic/` and `evals/dataset/synthetic/` support these tests. Evaluation source also supports methods tests, including calibration metrics. Generated results and collected handwriting belong outside the approved public file set. +### Offline tutoring episodes + +From the repository root, using the backend development environment: + +```bash +services/api/.venv/bin/python evals/tutoring.py --output /tmp/quack-tutoring.json +# Run the same evaluator against the earlier source version first to capture a baseline. +services/api/.venv/bin/python evals/tutoring.py --output /tmp/quack-after.json --baseline /tmp/quack-before.json +``` + +This reuses the existing HTTP test doubles with the real parser, isolated verifier, account-owned +state, hints, corrections, recap and practice routes. Four chain-rule episodes and a power-rule +control cover detection through follow-up practice. Five guard cases cover ambiguous readings, +disagreement, malformed output, timeout and zero budget. Results contain synthetic case IDs, +contract checks, synthetic practice questions and mock call counts. Shell model/database settings +are removed and unregistered HTTP calls are refused. No key or real model credits are used. +The comparison rejects changed fixtures, changed evaluator code and missing checks; a failed +episode returns a nonzero exit status. These episodes also run in ordinary backend regression tests. + +Chain-rule practice now follows a verified missing-factor pattern from My Mistakes or a recap. +It uses small sine, cosine and exponential questions with linear or power inner functions; +all keys are verified before display. Numerical angle offsets and powers of brackets are outside +this initial question bank. Existing uncertainty, completion and budget rules still apply. +The evaluator checks deterministic contracts and hint spoiler/fidelity filters. Human hint review, +real recognition accuracy, learning gains and production latency/cost require separate evidence. + ## What the tests establish Synthetic ink and scripted readings can verify mathematics, state transitions and UI flow. They do not establish physical Apple Pencil behaviour, real handwriting accuracy or learning outcomes. Supabase stand-ins do not establish correct configuration of a hosted Auth project. Two model readings can agree on a misrecognition. diff --git a/evals/tutoring.py b/evals/tutoring.py new file mode 100644 index 0000000..9606d44 --- /dev/null +++ b/evals/tutoring.py @@ -0,0 +1,260 @@ +"""Offline tutoring trajectories using the existing HTTP test doubles and real verifier. + +Run with the backend development Python: python evals/tutoring.py --output /tmp/tutoring.json +No real model, real handwriting, hosted database or student record is used. Assertions measure +software contracts, not recognition accuracy, the student's thought process or learning gains. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import random +import sys +import tempfile +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "services/api")) + +import httpx # noqa: E402 +import respx # noqa: E402 +from fastapi.testclient import TestClient # noqa: E402 + +# isort: split +from quack.main import build_services, create_app # noqa: E402 +from quack.services.hint_engine import find_problems # noqa: E402 +from quack.services.practice import generate_question # noqa: E402 +from tests.accounts_support import account_settings # noqa: E402 +from tests.test_learning import Model, Student # noqa: E402 +from tests.test_line_memory import compact # noqa: E402 +from tests.test_pipeline import CHAT, completion, png # noqa: E402 + + +@dataclass(frozen=True) +class Trajectory: + id: str + question: str + wrong: str + right: str + error: str + topic: str + seed: int + + +CASES = ( + Trajectory("sine_square", "d/dx(sin(x^2))", "cos(x^2)", "2x*cos(x^2)", + "chain_rule_inner_missing", "chain_rule", 0), + Trajectory("cosine_linear", "d/dx(cos(3x))", "-sin(3x)", "-3sin(3x)", + "chain_rule_inner_missing", "chain_rule", 1), + Trajectory("exponential_linear", "d/dx(exp(2x))", "exp(2x)", "2exp(2x)", + "chain_rule_inner_missing", "chain_rule", 2), + Trajectory("power_of_bracket", "d/dx((2x+1)^3)", "3(2x+1)^2", "6(2x+1)^2", + "chain_rule_inner_missing", "chain_rule", 3), + Trajectory("power_rule_control", "d/dx(x^3)", "2x^2", "3x^2", + "power_rule_coefficient", "power_rule", 4), +) + + +def isolated_client(directory: Path, **overrides: Any) -> TestClient: + # Explicit overrides beat shell credentials and prevent inherited database/route settings. + settings = account_settings( + directory, QUACK_DATABASE_URL="", SUPABASE_URL="", QUACK_LEDGER_BACKEND="file", + QUACK_ROUTES="", QUACK_PERSONALIZATION="off", **overrides, + ) + return TestClient(create_app(build_services(settings))) + + +class Script(Model): + def __init__(self) -> None: + super().__init__() + self.calls = 0 + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.calls += 1 + return super().__call__(request) + + +def evaluate_trajectory(case: Trajectory, directory: Path) -> dict[str, Any]: + checks: dict[str, bool] = {} + observations: dict[str, Any] = {} + model = Script() + # respx refuses every unregistered HTTP call. The only provider URL uses a fake key. + with respx.mock(assert_all_called=False) as mock, isolated_client(directory) as client: + mock.post(CHAT).mock(side_effect=model) + student = Student(client) + first = student.check(model, f"{case.question} = {case.wrong}") + mistakes = first["mistakes"] + checks["verified_error_and_category"] = bool( + len(mistakes) == 1 and mistakes[0]["verified"] and mistakes[0]["errorType"] == case.error + and first["intervention"]["shouldQuack"] + ) + if not mistakes: + return {"case": case.id, "checks": checks, "mock_provider_calls": model.calls} + mistake = mistakes[0]["id"] + checks["learning_accepts_detection"] = not student.event("mistake_detected", mistake)["refused"] + checks["duplicate_alert_suppressed"] = not student.check( + model, f"{case.question} = {case.wrong}", [mistake] + )["intervention"]["shouldQuack"] + before_hints = model.calls + hints = [client.post("/api/hint", json={ + "sessionId": student.page, "mistakeId": mistake, "level": level, + "studySessionId": student.session, + }, headers=student.headers).json() for level in (1, 2)] + checks["bounded_prepared_hints"] = model.calls == before_hints and all(h["source"] == "template" for h in hints) + # Reuse the product's spoiler and fidelity checks against its private verified record. + user = client.get("/api/profile", headers=student.headers).json()["userId"] + tracked = client.app.state.services.mistakes.get_mistake(f"{user}:{student.page}", mistake) + checks["hint_fidelity_and_no_spoiler"] = tracked is not None and all( + not find_problems(level, hint["text"], tracked) for level, hint in enumerate(hints, 1) + ) + fixed = student.check(model, f"{case.question} = {case.right}", [mistake]) + checks["correction_verified"] = fixed["resolvedMistakeIds"] == [mistake] and not fixed["mistakes"] + checks["learning_accepts_correction"] = not student.event("correction_verified", mistake)["refused"] + student.event("correction_verified", mistake) # Retries cannot inflate progress. + entry = client.get(f"/api/mistakes/{mistake}", headers=student.headers).json() + checks["mistake_offers_targeted_practice"] = entry["practiceTopic"] == case.topic + recap = student.finish() + checks["counts_once_with_help"] = ( + recap["glance"]["found"] == recap["glance"]["corrected"] == recap["glance"]["assisted"] == 1 + ) + recommendation = recap["recommendation"] + checks["recap_offers_targeted_practice"] = recommendation is not None and recommendation["topic"] == case.topic + # Pin question generation without changing IDs or any model/verifier behaviour. + with patch("quack.services.practice.generate_question", side_effect=lambda topic, difficulty, rng: generate_question( + topic, difficulty, random.Random(case.seed) + )): + before = model.calls + response = client.post("/api/practice/sessions", json={ + "topic": case.topic, "originSessionId": student.session, "sourceMistakeId": mistake, + }, headers=student.headers) + checks["verified_practice_available_without_model"] = response.status_code == 200 and before == model.calls + checks["practice_wrong_then_correct"] = False + checks["practice_completion_counted_once"] = False + checks["answer_hidden_on_creation"] = False + if response.status_code == 200: + problem = response.json() + user = client.get("/api/profile", headers=student.headers).json()["userId"] + row = client.app.state.services.accounts.store.problem(user, problem["id"]) + checks["answer_hidden_on_creation"] = row["expected_answer"] not in json.dumps(problem) + + def answer(text: str, version: int, known: list[str]) -> dict[str, Any]: + model.reply = [compact(text)] + result = client.post(f"/api/practice/problems/{problem['id']}/check", json={ + "sessionId": problem["checkSessionId"], "snapshotVersion": version, + "image": png(), "acknowledgedMistakeIds": known, + "inkLines": [{"id": f"practice-{version}", "inImage": True}], + }, headers=student.headers) + result.raise_for_status() + return result.json() + + wrong = answer(f"2*({row['expected_answer']})", 1, []) + known = [m["id"] for m in wrong["mistakes"]] + right = answer(row["expected_answer"], 2, known) + observations["practice"] = { + "question": row["prompt_math"], "wrong_outcome": wrong["outcome"], "right_outcome": right["outcome"], + "wrong_status": wrong["status"], "right_status": right["status"], + } + answer(row["expected_answer"], 3, []) + checks["practice_wrong_then_correct"] = wrong["outcome"] == "incorrect" and right["outcome"] == "correct" + summary = client.post(f"/api/practice/sessions/{problem['sessionId']}/finish", headers=student.headers).json() + checks["practice_completion_counted_once"] = summary["correct"] == summary["attempted"] == 1 + checks["bounded_reading_calls"] = model.calls <= 12 + return {"case": case.id, "checks": checks, "observations": observations, "mock_provider_calls": model.calls} + + +SAFETY_CASES = ("ambiguous", "disagreement", "malformed", "timeout", "zero_budget") + + +def evaluate_safety(case: str, directory: Path) -> dict[str, Any]: + overrides = {"QUACK_BUDGET_PROJECT_USD": 0} if case == "zero_budget" else {} + with respx.mock(assert_all_called=False) as mock, isolated_client(directory, **overrides) as client: + student = Student(client) + wrong = compact("d/dx(sin(x^2)) = cos(x^2)") + route = mock.post(CHAT) + if case == "timeout": + route.mock(side_effect=httpx.ReadTimeout("synthetic timeout")) + elif case == "malformed": + route.mock(return_value=completion({"not_lines": []})) + elif case == "disagreement": + right = compact("d/dx(sin(x^2)) = 2x*cos(x^2)") + route.mock(side_effect=[completion({"lines": [wrong]}), completion({"lines": [right]})]) + else: + if case == "ambiguous": + wrong.update(legibility="unsure", alternatives=["d/dx(sin(x^2)) = 2x*cos(x^2)"]) + route.mock(return_value=completion({"lines": [wrong]})) + reply = client.post("/api/analyze", headers=student.headers, json={ + "sessionId": student.page, "snapshotVersion": 1, "image": png(), + "inkLines": [{"id": "guard", "inImage": True}], "studySessionId": student.session, + }).json() + checks = { + "no_false_alert": not reply["mistakes"] and not reply["intervention"]["shouldQuack"], + "no_learning_penalty": student.finish()["glance"]["found"] == 0, + # Schema validation permits one repair per reading; timeouts do not retry. + "bounded_reading_calls": route.call_count <= (4 if case == "malformed" else 2), + } + if case == "zero_budget": + checks["budget_blocks_provider"] = route.call_count == 0 + return {"case": case, "checks": checks, "mock_provider_calls": route.call_count} + + +def run() -> dict[str, Any]: + rows = [] + # Match the release runner: shell credentials, proxy configuration and routes are irrelevant. + env = {key: value for key, value in os.environ.items() if not key.startswith(("MODEL_", "QUACK_", "SUPABASE_")) + and key.upper() not in {"HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"}} + with patch.dict(os.environ, env, clear=True), tempfile.TemporaryDirectory(prefix="quack-tutoring-") as temp: + for case in CASES: + directory = Path(temp) / case.id + directory.mkdir() + rows.append(evaluate_trajectory(case, directory)) + for guard in SAFETY_CASES: + directory = Path(temp) / guard + directory.mkdir() + rows.append(evaluate_safety(guard, directory)) + fixtures = json.dumps({"trajectories": [asdict(case) for case in CASES], "safety": SAFETY_CASES}, sort_keys=True) + return {"version": 1, "mode": "offline-scripted-readings", "live_model_calls": 0, + "fixtures_sha256": hashlib.sha256(fixtures.encode()).hexdigest(), + "evaluator_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "rows": rows, "passed_cases": sum(all(row["checks"].values()) for row in rows), "cases": len(rows)} + + +def compare(baseline: dict[str, Any], current: dict[str, Any]) -> dict[str, Any]: + """Reject incomparable evidence, including changed fixtures or silently missing checks.""" + + if any(baseline[key] != current[key] for key in ("version", "mode", "fixtures_sha256", "evaluator_sha256")): + raise ValueError("baseline uses a different evaluation method or fixtures") + before = {row["case"]: row["checks"] for row in baseline["rows"]} + after = {row["case"]: row["checks"] for row in current["rows"]} + if before.keys() != after.keys() or any(before[case].keys() != after[case].keys() for case in before): + raise ValueError("baseline and current results must contain the same cases and checks") + changed = [{"case": case, "check": check, "before": old, "after": after[case][check]} + for case, checks in before.items() for check, old in checks.items() if old != after[case][check]] + return {"before_passed_cases": baseline["passed_cases"], "after_passed_cases": current["passed_cases"], + "changed_checks": changed, "regressions": [item for item in changed if not item["after"]]} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--baseline", type=Path, help="compare with a report produced by this same evaluator") + args = parser.parse_args() + result = run() + if args.baseline: + result["comparison"] = compare(json.loads(args.baseline.read_text()), result) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n") + for row in result["rows"]: + failures = [name for name, passed in row["checks"].items() if not passed] + print(f"{row['case']}: {'PASS' if not failures else ', '.join(failures)}") + print(f"{result['passed_cases']}/{result['cases']} trajectories passed; no live model calls.") + return 0 if result["passed_cases"] == result["cases"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/public-files.txt b/scripts/public-files.txt index dfe2a9f..b9f3bbb 100644 --- a/scripts/public-files.txt +++ b/scripts/public-files.txt @@ -441,3 +441,6 @@ services/api/requirements-tooling.lock services/api/tests/test_release_safety.py services/api/tests/fixtures/recorded-routing.json scripts/check_browser_results.py +evals/tutoring.py +services/api/tests/test_chain_rule_practice.py +services/api/tests/test_tutoring_eval.py diff --git a/services/api/quack/schemas.py b/services/api/quack/schemas.py index 0b659c2..9aea373 100644 --- a/services/api/quack/schemas.py +++ b/services/api/quack/schemas.py @@ -493,7 +493,7 @@ class SessionHistory(ApiModel): PracticeTopic = Literal[ "addition_subtraction", "multiplication", "division", "expanding_brackets", "combining_like_terms", - "solving_equations", "power_rule", "sum_and_constant_rules", "integration", + "solving_equations", "power_rule", "sum_and_constant_rules", "integration", "chain_rule", ] PracticeOutcome = Literal["in_progress", "incorrect", "correct"] diff --git a/services/api/quack/services/practice.py b/services/api/quack/services/practice.py index 58d878e..8022349 100644 --- a/services/api/quack/services/practice.py +++ b/services/api/quack/services/practice.py @@ -31,6 +31,7 @@ "power_rule", "sum_and_constant_rules", "integration", + "chain_rule", ) _SUPERSCRIPTS = str.maketrans("0123456789", "⁰¹²³⁴⁵⁶⁷⁸⁹") _SUBSCRIPTS = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉") @@ -198,6 +199,25 @@ def _sum_rules(rng: random.Random, difficulty: str) -> Question: ) +def _chain_rule(rng: random.Random, difficulty: str) -> Question: + """An outside function and a nontrivial inside derivative, with an explicit answer key. + + Basic questions use ax; intermediate ones use ax^n. Avoid numerical angle offsets: + the existing verifier deliberately abstains when their units are unclear. Symbolic + work and answer completion must approve the key before the question can be shown. + """ + + power = 1 if difficulty == "basic" else rng.randint(2, 3) + scale = rng.randint(2, 4) + inner = _polynomial({power: scale}) + rate = _polynomial({power - 1: scale * power}) + family = rng.choice(("sin", "cos", "exp")) + expression = f"{family}({inner})" + outside = {"sin": f"cos({inner})", "cos": f"-sin({inner})", "exp": f"exp({inner})"}[family] + expected = f"({rate})*({outside})" + return _question("chain_rule", difficulty, f"d/dx({expression})", expected, "differentiation") + + _TEMPLATES = { "addition_subtraction": _adding, "solving_equations": _equations, @@ -208,6 +228,7 @@ def _sum_rules(rng: random.Random, difficulty: str) -> Question: "combining_like_terms": _like_terms, "power_rule": _power_rule, "sum_and_constant_rules": _sum_rules, + "chain_rule": _chain_rule, } diff --git a/services/api/quack/services/topics.py b/services/api/quack/services/topics.py index b1ca85c..ff43207 100644 --- a/services/api/quack/services/topics.py +++ b/services/api/quack/services/topics.py @@ -40,7 +40,7 @@ class Topic: Topic("power_rule", "Power rule", CALCULUS, "power_rule"), Topic("sum_and_constant_rules", "Sum and constant rules", CALCULUS, "sum_and_constant_rules"), Topic("standard_derivatives", "Derivatives of sin, cos and other functions", CALCULUS), - Topic("chain_rule", "Chain rule", CALCULUS), + Topic("chain_rule", "Chain rule", CALCULUS, "chain_rule"), Topic("product_rule", "Product rule", CALCULUS), Topic("quotient_rule", "Quotient rule", CALCULUS), Topic("integration", "Integration", CALCULUS, "integration"), diff --git a/services/api/scripts/adaptive_eval.py b/services/api/scripts/adaptive_eval.py index 39a92fa..1b681ed 100644 --- a/services/api/scripts/adaptive_eval.py +++ b/services/api/scripts/adaptive_eval.py @@ -17,7 +17,7 @@ from dataclasses import dataclass from quack.services.adaptive import Attempt, choose_next, normal_progression -from quack.services.curriculum import load_all +from quack.services.curriculum import course from quack.services.lesson_questions import TEMPLATES, build from quack.services.math_verifier import LineInput, verify_lines from quack.services.misconceptions import TAXONOMY, classify @@ -59,8 +59,9 @@ class Case: def run_case(case: Case) -> dict[str, str]: - course = load_all()[0] - level = course.level(case.level) + algebra = course("algebra_foundations") + assert algebra is not None + level = algebra.level(case.level) assert level is not None template, params = case.asked or (level.opener.template, dict(level.opener.params)) # type: ignore[union-attr] question = build(template, random.Random(0), dict(params)) diff --git a/services/api/tests/test_chain_rule_practice.py b/services/api/tests/test_chain_rule_practice.py new file mode 100644 index 0000000..b863bf5 --- /dev/null +++ b/services/api/tests/test_chain_rule_practice.py @@ -0,0 +1,108 @@ +"""Chain-rule practice keys, missing-factor patterns and existing safety boundaries. No paid calls.""" + +from __future__ import annotations + +import asyncio +import random +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import pytest +import sympy as sp +from fastapi.testclient import TestClient + +from quack.services.math_parser import parse_expression, to_sympy +from quack.services.math_verifier import LineInput, completes_answer, verify_lines +from quack.services.practice import MAX_TRIES, generate_question, new_question +from quack.services.verifier_pool import VerificationFailed +from tests.accounts_support import account_settings, register +from tests.test_line_memory import compact +from tests.test_pipeline import png +from tests.test_practice import Model, Practice, model # noqa: F401 + + +@pytest.mark.parametrize("difficulty", ("basic", "intermediate")) +@pytest.mark.parametrize("seed", range(24)) +def test_independent_keys_equivalent_answers_and_missing_inner_factors(difficulty: str, seed: int) -> None: + question = generate_question("chain_rule", difficulty, random.Random(seed)) + source = to_sympy(parse_expression(question.math[5:-1])) + x = next(iter(source.free_symbols)) + expected = to_sympy(parse_expression(question.expected)) + assert sp.simplify(sp.diff(source, x) - expected) == 0 + # Equivalent products in a different written order must be accepted and finish the question. + answer = str(sp.diff(source, x)).replace("**", "^") + statement = f"{question.math} = {answer}" + assert verify_lines([LineInput(0, statement, posed=True)]).status == "valid" + assert completes_answer(question.math, question.expected, statement) + inner = source.base if source.is_Pow else source.args[0] + rate = sp.diff(inner, x) + assert rate != 1 + missing = str(sp.simplify(expected / rate)).replace("**", "^") + report = verify_lines([LineInput(0, f"{question.math} = {missing}", posed=True)]) + assert report.first_invalid is not None + assert report.first_invalid.error_type == "chain_rule_inner_missing" + assert not completes_answer(question.math, question.expected, f"= {missing}") + + +@pytest.mark.parametrize("difficulty", ("basic", "intermediate")) +def test_seeded_fixtures_cover_all_families_and_difficulty(difficulty: str) -> None: + families = set() + for seed in range(24): + source = to_sympy(parse_expression(generate_question("chain_rule", difficulty, random.Random(seed)).math[5:-1])) + families.add(source.func) + inner = source.base if source.is_Pow else source.args[0] + x = next(iter(source.free_symbols)) + expected_degrees = (1,) if difficulty == "basic" else (2, 3) + assert sp.degree(inner, x) in expected_degrees + assert families == {sp.sin, sp.cos, sp.exp} + + +def test_uncertain_practice_does_not_finish_or_penalise(accounts_client: TestClient, model: Model) -> None: # noqa: F811 + practice = Practice(accounts_client, "chain_rule") + model.reply = [compact(practice.row["expected_answer"], legibility="unsure", alternatives=["0"])] + reply = accounts_client.post(f"/api/practice/problems/{practice.problem['id']}/check", headers=practice.headers, json={ + "sessionId": practice.problem["checkSessionId"], "snapshotVersion": 1, + "image": png(), + "inkLines": [{"id": "unsure", "inImage": True}], + }).json() + assert reply["outcome"] == "in_progress" + assert reply["mistakes"] == [] + assert practice.row["status"] == "open" + assert practice.summary()["correct"] == 0 + + +def test_uncheckable_questions_fail_closed_without_model_calls(accounts_client: TestClient, model: Model) -> None: # noqa: F811 + headers = register(accounts_client) + failure = AsyncMock(side_effect=VerificationFailed("timeout")) + with patch("quack.services.practice.shared_pool.verify", failure): + reply = accounts_client.post("/api/practice/sessions", headers=headers, json={"topic": "chain_rule"}) + assert reply.status_code == 503 + assert failure.await_count == MAX_TRIES + assert not model.calls + + +async def test_cancelled_generation_does_not_retry() -> None: + cancelled = AsyncMock(side_effect=asyncio.CancelledError) + with patch("quack.services.practice.shared_pool.verify", cancelled), pytest.raises(asyncio.CancelledError): + await new_question("chain_rule", "basic", set(), 4) + assert cancelled.await_count == 1 + + +def test_no_model_for_generation_and_zero_budget_blocks_reading(tmp_path: Path, model: Model) -> None: # noqa: F811 + from quack.main import build_services, create_app + + client = TestClient(create_app(build_services(account_settings(tmp_path, QUACK_BUDGET_PROJECT_USD=0)))) + practice = Practice(client, "chain_rule") + assert model.calls == [] + reply = practice.answer(model, practice.row["expected_answer"]) + assert model.calls == [] + assert reply["outcome"] == "in_progress" + assert reply["modelIssue"]["category"] == "budget_exceeded" + assert practice.row["status"] == "open" + + +def test_another_account_cannot_reveal_chain_rule_key(accounts_client: TestClient) -> None: + practice = Practice(accounts_client, "chain_rule") + other = register(accounts_client, "other@example.com") + reply = accounts_client.post(f"/api/practice/problems/{practice.problem['id']}/reveal", headers=other) + assert reply.status_code == 404 diff --git a/services/api/tests/test_practice.py b/services/api/tests/test_practice.py index 49d44a5..d97a128 100644 --- a/services/api/tests/test_practice.py +++ b/services/api/tests/test_practice.py @@ -43,6 +43,10 @@ def wrong_answer(expected: str) -> str: if expected.startswith("x ="): return f"x = {int(expected[3:]) + 1}" + if any(function in expected for function in ("sin(", "cos(", "exp(")): + # A constant offset can disappear relative to a large exponential at sample points. + # A scaling error stays distinct; the verifier's conservative comparison is unchanged. + return f"2*({expected})" return f"{expected} + x" if "C" in expected else f"{expected} + 1" @@ -324,7 +328,7 @@ def test_practice_needs_signing_in(self, accounts_client: TestClient) -> None: assert accounts_client.post("/api/practice/sessions", json={"topic": "division"}).status_code == 401 def test_unsupported_topics_are_refused(self, accounts_client: TestClient) -> None: - response = accounts_client.post("/api/practice/sessions", json={"topic": "chain_rule"}, headers=register(accounts_client)) + response = accounts_client.post("/api/practice/sessions", json={"topic": "product_rule"}, headers=register(accounts_client)) assert response.status_code == 422 diff --git a/services/api/tests/test_tutoring_eval.py b/services/api/tests/test_tutoring_eval.py new file mode 100644 index 0000000..c1fa179 --- /dev/null +++ b/services/api/tests/test_tutoring_eval.py @@ -0,0 +1,50 @@ +"""The offline trajectory scorecard is also part of the existing regression suite.""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "evals")) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +from adaptive_eval import CASES as ADAPTIVE_CASES # noqa: E402 +from adaptive_eval import run_case # noqa: E402 +from tutoring import CASES, SAFETY_CASES, Trajectory, compare, evaluate_safety, evaluate_trajectory # noqa: E402 + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: case.id) +def test_complete_tutoring_trajectory(case: Trajectory, tmp_path: Path) -> None: + result = evaluate_trajectory(case, tmp_path) + assert all(result["checks"].values()), result + + +@pytest.mark.parametrize("case", SAFETY_CASES) +def test_episode_safety(case: str, tmp_path: Path) -> None: + result = evaluate_safety(case, tmp_path) + assert all(result["checks"].values()), result + + +def report(checks: dict[str, bool]) -> dict: + return {"version": 1, "mode": "offline", "fixtures_sha256": "fixture", "evaluator_sha256": "method", + "passed_cases": int(all(checks.values())), "rows": [{"case": "case", "checks": checks}]} + + +def test_comparison_exposes_regressions_and_refuses_missing_checks() -> None: + before = report({"safe": True, "followup": False}) + after = report({"safe": False, "followup": True}) + assert compare(before, after)["regressions"] == [{"case": "case", "check": "safe", "before": True, "after": False}] + with pytest.raises(ValueError, match="same cases and checks"): + compare(before, report({"safe": True})) + + +def test_changed_fixture_or_method_cannot_claim_an_improvement() -> None: + before = report({"safe": True}) + with pytest.raises(ValueError, match="different evaluation"): + compare(before, before | {"fixtures_sha256": "different"}) + + +def test_existing_adaptive_report_selects_algebra_independently_of_catalog_order() -> None: + rows = [run_case(case) for case in ADAPTIVE_CASES] + assert len(rows) == 14 + assert all(row["verdict"] != "not a mistake" and row["fits"] == "yes" for row in rows) From 01c3546b8c438cd289810acce4961050e87a4003 Mon Sep 17 00:00:00 2001 From: Brendan-Lim <121050815+Brendan-Lim@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:38:31 +0800 Subject: [PATCH 2/2] Draw exponential practice answers in synthetic browser fixtures The chain-rule browser journey can select exp(ax), but the synthetic alphabet had no e or p. Add those two glyphs without changing existing strokes, application behavior or verification rules. Render every distinct generated chain-rule question and answer in a regression so all families remain drawable regardless of random selection. The original hosted failure is retained; all three local browser repetitions and 69 focused offline tests pass before rerunning the full release gate. --- evals/synthetic/glyphs.json | 2 +- .../api/tests/test_chain_rule_practice.py | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/evals/synthetic/glyphs.json b/evals/synthetic/glyphs.json index b7b9a8c..ce2c76f 100644 --- a/evals/synthetic/glyphs.json +++ b/evals/synthetic/glyphs.json @@ -1 +1 @@ -{"_comment": "A hand-authored single-stroke alphabet in a unit box (x right, y down). Used only to draw synthetic test input. It is not real handwriting.", "0": [[[0.5, 0], [0.15, 0.15], [0.05, 0.5], [0.15, 0.85], [0.5, 1], [0.85, 0.85], [0.95, 0.5], [0.85, 0.15], [0.5, 0]]], "1": [[[0.25, 0.2], [0.5, 0], [0.5, 1]]], "2": [[[0.1, 0.25], [0.25, 0.05], [0.55, 0], [0.85, 0.15], [0.85, 0.4], [0.6, 0.65], [0.1, 1], [0.95, 1]]], "3": [[[0.1, 0.1], [0.45, 0], [0.85, 0.12], [0.85, 0.35], [0.45, 0.48], [0.9, 0.62], [0.9, 0.88], [0.45, 1], [0.1, 0.9]]], "4": [[[0.7, 1], [0.7, 0], [0.05, 0.65], [0.95, 0.65]]], "5": [[[0.85, 0], [0.2, 0], [0.15, 0.45], [0.55, 0.4], [0.9, 0.6], [0.85, 0.88], [0.45, 1], [0.1, 0.9]]], "6": [[[0.8, 0.05], [0.4, 0.1], [0.12, 0.5], [0.12, 0.8], [0.45, 1], [0.85, 0.85], [0.85, 0.55], [0.5, 0.42], [0.14, 0.6]]], "7": [[[0.08, 0], [0.92, 0], [0.4, 1]]], "8": [[[0.5, 0.48], [0.18, 0.3], [0.2, 0.1], [0.5, 0], [0.8, 0.1], [0.82, 0.3], [0.5, 0.48], [0.12, 0.68], [0.15, 0.9], [0.5, 1], [0.85, 0.9], [0.88, 0.68], [0.5, 0.48]]], "9": [[[0.85, 0.35], [0.5, 0.5], [0.15, 0.35], [0.18, 0.12], [0.5, 0], [0.82, 0.12], [0.85, 0.35], [0.8, 0.7], [0.45, 1]]], "x": [[[0.1, 0.35], [0.9, 1]], [[0.9, 0.35], [0.1, 1]]], "y": [[[0.1, 0.35], [0.5, 0.85]], [[0.9, 0.35], [0.25, 1.25]]], "d": [[[0.75, 0.5], [0.4, 0.42], [0.1, 0.62], [0.12, 0.9], [0.4, 1], [0.75, 0.82]], [[0.78, 0], [0.78, 1]]], "+": [[[0.1, 0.55], [0.9, 0.55]], [[0.5, 0.2], [0.5, 0.9]]], "-": [[[0.12, 0.55], [0.88, 0.55]]], "=": [[[0.12, 0.42], [0.88, 0.42]], [[0.12, 0.7], [0.88, 0.7]]], "(": [[[0.7, 0], [0.32, 0.28], [0.28, 0.72], [0.7, 1]]], ")": [[[0.3, 0], [0.68, 0.28], [0.72, 0.72], [0.3, 1]]], "/": [[[0.8, 0], [0.2, 1]]], "s": [[[0.8, 0.42], [0.45, 0.35], [0.18, 0.48], [0.3, 0.65], [0.72, 0.75], [0.8, 0.9], [0.5, 1], [0.15, 0.92]]], "i": [[[0.5, 0.4], [0.5, 1]], [[0.5, 0.15], [0.5, 0.2]]], "n": [[[0.15, 0.38], [0.15, 1]], [[0.15, 0.55], [0.45, 0.38], [0.8, 0.5], [0.82, 1]]], "c": [[[0.82, 0.48], [0.45, 0.36], [0.15, 0.55], [0.15, 0.82], [0.45, 1], [0.82, 0.9]]], "o": [[[0.5, 0.36], [0.15, 0.5], [0.12, 0.8], [0.5, 1], [0.86, 0.82], [0.85, 0.52], [0.5, 0.36]]], "t": [[[0.45, 0.05], [0.45, 0.88], [0.8, 1]], [[0.12, 0.4], [0.8, 0.4]]], "a": [[[0.78, 0.42], [0.4, 0.36], [0.12, 0.6], [0.2, 0.92], [0.5, 1], [0.78, 0.8]], [[0.8, 0.38], [0.82, 1]]], "<": [[[0.85, 0.25], [0.15, 0.6], [0.85, 0.95]]], "f": [[[0.8, 0.08], [0.5, 0], [0.38, 0.2], [0.38, 1.1]], [[0.12, 0.42], [0.72, 0.42]]], "'": [[[0.5, 0], [0.42, 0.25]]], ",": [[[0.5, 0.9], [0.38, 1.1]]], ".": [[[0.48, 0.96], [0.5, 1]]], "∫": [[[0.8, 0.0], [0.55, 0.02], [0.45, 0.3], [0.42, 1.0], [0.3, 1.25], [0.1, 1.22]]], "√": [[[0.0, 0.62], [0.18, 0.55], [0.38, 1.0], [0.6, 0.0], [1.05, 0.0]]], "*": [[[0.38, 0.42], [0.62, 0.68]], [[0.62, 0.42], [0.38, 0.68]]], "÷": [[[0.3, 0.55], [0.7, 0.55]], [[0.5, 0.38], [0.5, 0.41]], [[0.5, 0.69], [0.5, 0.72]]], "×": [[[0.38, 0.42], [0.62, 0.68]], [[0.62, 0.42], [0.38, 0.68]]]} \ No newline at end of file +{"_comment": "A hand-authored single-stroke alphabet in a unit box (x right, y down). Used only to draw synthetic test input. It is not real handwriting.", "0": [[[0.5, 0], [0.15, 0.15], [0.05, 0.5], [0.15, 0.85], [0.5, 1], [0.85, 0.85], [0.95, 0.5], [0.85, 0.15], [0.5, 0]]], "1": [[[0.25, 0.2], [0.5, 0], [0.5, 1]]], "2": [[[0.1, 0.25], [0.25, 0.05], [0.55, 0], [0.85, 0.15], [0.85, 0.4], [0.6, 0.65], [0.1, 1], [0.95, 1]]], "3": [[[0.1, 0.1], [0.45, 0], [0.85, 0.12], [0.85, 0.35], [0.45, 0.48], [0.9, 0.62], [0.9, 0.88], [0.45, 1], [0.1, 0.9]]], "4": [[[0.7, 1], [0.7, 0], [0.05, 0.65], [0.95, 0.65]]], "5": [[[0.85, 0], [0.2, 0], [0.15, 0.45], [0.55, 0.4], [0.9, 0.6], [0.85, 0.88], [0.45, 1], [0.1, 0.9]]], "6": [[[0.8, 0.05], [0.4, 0.1], [0.12, 0.5], [0.12, 0.8], [0.45, 1], [0.85, 0.85], [0.85, 0.55], [0.5, 0.42], [0.14, 0.6]]], "7": [[[0.08, 0], [0.92, 0], [0.4, 1]]], "8": [[[0.5, 0.48], [0.18, 0.3], [0.2, 0.1], [0.5, 0], [0.8, 0.1], [0.82, 0.3], [0.5, 0.48], [0.12, 0.68], [0.15, 0.9], [0.5, 1], [0.85, 0.9], [0.88, 0.68], [0.5, 0.48]]], "9": [[[0.85, 0.35], [0.5, 0.5], [0.15, 0.35], [0.18, 0.12], [0.5, 0], [0.82, 0.12], [0.85, 0.35], [0.8, 0.7], [0.45, 1]]], "x": [[[0.1, 0.35], [0.9, 1]], [[0.9, 0.35], [0.1, 1]]], "y": [[[0.1, 0.35], [0.5, 0.85]], [[0.9, 0.35], [0.25, 1.25]]], "d": [[[0.75, 0.5], [0.4, 0.42], [0.1, 0.62], [0.12, 0.9], [0.4, 1], [0.75, 0.82]], [[0.78, 0], [0.78, 1]]], "+": [[[0.1, 0.55], [0.9, 0.55]], [[0.5, 0.2], [0.5, 0.9]]], "-": [[[0.12, 0.55], [0.88, 0.55]]], "=": [[[0.12, 0.42], [0.88, 0.42]], [[0.12, 0.7], [0.88, 0.7]]], "(": [[[0.7, 0], [0.32, 0.28], [0.28, 0.72], [0.7, 1]]], ")": [[[0.3, 0], [0.68, 0.28], [0.72, 0.72], [0.3, 1]]], "/": [[[0.8, 0], [0.2, 1]]], "s": [[[0.8, 0.42], [0.45, 0.35], [0.18, 0.48], [0.3, 0.65], [0.72, 0.75], [0.8, 0.9], [0.5, 1], [0.15, 0.92]]], "i": [[[0.5, 0.4], [0.5, 1]], [[0.5, 0.15], [0.5, 0.2]]], "n": [[[0.15, 0.38], [0.15, 1]], [[0.15, 0.55], [0.45, 0.38], [0.8, 0.5], [0.82, 1]]], "c": [[[0.82, 0.48], [0.45, 0.36], [0.15, 0.55], [0.15, 0.82], [0.45, 1], [0.82, 0.9]]], "o": [[[0.5, 0.36], [0.15, 0.5], [0.12, 0.8], [0.5, 1], [0.86, 0.82], [0.85, 0.52], [0.5, 0.36]]], "t": [[[0.45, 0.05], [0.45, 0.88], [0.8, 1]], [[0.12, 0.4], [0.8, 0.4]]], "a": [[[0.78, 0.42], [0.4, 0.36], [0.12, 0.6], [0.2, 0.92], [0.5, 1], [0.78, 0.8]], [[0.8, 0.38], [0.82, 1]]], "<": [[[0.85, 0.25], [0.15, 0.6], [0.85, 0.95]]], "f": [[[0.8, 0.08], [0.5, 0], [0.38, 0.2], [0.38, 1.1]], [[0.12, 0.42], [0.72, 0.42]]], "'": [[[0.5, 0], [0.42, 0.25]]], ",": [[[0.5, 0.9], [0.38, 1.1]]], ".": [[[0.48, 0.96], [0.5, 1]]], "\u222b": [[[0.8, 0.0], [0.55, 0.02], [0.45, 0.3], [0.42, 1.0], [0.3, 1.25], [0.1, 1.22]]], "\u221a": [[[0.0, 0.62], [0.18, 0.55], [0.38, 1.0], [0.6, 0.0], [1.05, 0.0]]], "*": [[[0.38, 0.42], [0.62, 0.68]], [[0.62, 0.42], [0.38, 0.68]]], "\u00f7": [[[0.3, 0.55], [0.7, 0.55]], [[0.5, 0.38], [0.5, 0.41]], [[0.5, 0.69], [0.5, 0.72]]], "\u00d7": [[[0.38, 0.42], [0.62, 0.68]], [[0.62, 0.42], [0.38, 0.68]]], "e": [[[0.15, 0.66], [0.82, 0.66], [0.76, 0.46], [0.48, 0.36], [0.19, 0.48], [0.12, 0.76], [0.28, 0.94], [0.55, 1], [0.84, 0.89]]], "p": [[[0.16, 0.38], [0.16, 1.25]], [[0.16, 0.5], [0.46, 0.36], [0.8, 0.48], [0.84, 0.73], [0.64, 0.91], [0.35, 0.89], [0.16, 0.75]]]} diff --git a/services/api/tests/test_chain_rule_practice.py b/services/api/tests/test_chain_rule_practice.py index b863bf5..a984f85 100644 --- a/services/api/tests/test_chain_rule_practice.py +++ b/services/api/tests/test_chain_rule_practice.py @@ -4,6 +4,7 @@ import asyncio import random +import sys from pathlib import Path from unittest.mock import AsyncMock, patch @@ -57,6 +58,25 @@ def test_seeded_fixtures_cover_all_families_and_difficulty(difficulty: str) -> N assert families == {sp.sin, sp.cos, sp.exp} +def test_all_generated_questions_and_answers_have_drawable_synthetic_glyphs() -> None: + """The browser draws from this same alphabet; exp must work whichever question is chosen.""" + + sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "evals")) + from synthetic.render import GLYPHS, render + + for difficulty, count in (("basic", 9), ("intermediate", 18)): + questions = { + question.math: question + for question in (generate_question("chain_rule", difficulty, random.Random(seed)) for seed in range(1000)) + } + assert len(questions) == count + for question in questions.values(): + text = f"{question.math} = {question.expected}" + assert not set(text) - set(GLYPHS) - {" ", "^"}, text + rendered = render([{"text": text}]) + assert rendered.getextrema()[0][0] < 255 # It drew ink, rather than a blank fixture. + + def test_uncertain_practice_does_not_finish_or_penalise(accounts_client: TestClient, model: Model) -> None: # noqa: F811 practice = Practice(accounts_client, "chain_rule") model.reply = [compact(practice.row["expected_answer"], legibility="unsure", alternatives=["0"])]