diff --git a/.dockerignore b/.dockerignore index f9327cc..03bb1f1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,6 +11,7 @@ !apps/web/public/ !services/api/quack/ !apps/web/app/collect/ +!apps/web/app/fonts/ !apps/web/components/account/ !apps/web/components/ask/ !apps/web/components/collect/ @@ -29,6 +30,8 @@ !apps/web/.gitignore !apps/web/app/collect/page.tsx !apps/web/app/favicon.ico +!apps/web/app/fonts/bricolage-grotesque-latin-700.woff2 +!apps/web/app/fonts/figtree-latin-variable.woff2 !apps/web/app/globals.css !apps/web/app/layout.tsx !apps/web/app/page.tsx @@ -121,6 +124,7 @@ !services/api/quack/llm/report.py !services/api/quack/llm/router.py !services/api/quack/main.py +!services/api/quack/observability.py !services/api/quack/prompts/__init__.py !services/api/quack/prompts/ask.v1.md !services/api/quack/prompts/explain.v1.md diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..63c606d --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,86 @@ +name: Release checks + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: verify-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 60 + services: + postgres: + image: postgres:18 + env: + POSTGRES_USER: quack_test + POSTGRES_PASSWORD: test-only-password + POSTGRES_DB: quack_ledger_test + ports: ["5432:5432"] + options: >- + --health-cmd "pg_isready -U quack_test -d quack_ledger_test" + --health-interval 5s --health-timeout 5s --health-retries 10 + env: + CI: "true" + QUACK_ENV: production + QUACK_AUTH_MODE: "off" + MODEL_API_KEY: "" + QUACK_LEDGER_TEST_DATABASE_URL: postgresql://quack_test:test-only-password@127.0.0.1:5432/quack_ledger_test + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: | + services/api/requirements.lock + services/api/requirements-tooling.lock + services/api/requirements*.txt + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: apps/web/package-lock.json + - name: Install backend + run: | + python -m venv services/api/.venv + services/api/.venv/bin/python -m pip install -r services/api/requirements.lock -r services/api/requirements-tooling.lock -r services/api/requirements-dev.txt + - name: Install web + working-directory: apps/web + run: npm ci + - name: Audit locked Python dependencies + run: services/api/.venv/bin/python -m pip_audit --no-deps --disable-pip -r services/api/requirements.lock -r services/api/requirements-tooling.lock + - name: Audit web dependencies + working-directory: apps/web + run: npm audit + - name: Install test browser + working-directory: apps/web + run: npx playwright install --with-deps chromium + - name: Install checksum-verified secret scanner + run: | + mkdir -p "$RUNNER_TEMP/quack-tools" + curl --fail --silent --show-error --location https://github.com/gitleaks/gitleaks/releases/download/v8.30.1/gitleaks_8.30.1_linux_x64.tar.gz -o "$RUNNER_TEMP/quack-tools/gitleaks.tar.gz" + echo "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb $RUNNER_TEMP/quack-tools/gitleaks.tar.gz" | sha256sum --check + tar -xzf "$RUNNER_TEMP/quack-tools/gitleaks.tar.gz" -C "$RUNNER_TEMP/quack-tools" gitleaks + echo "$RUNNER_TEMP/quack-tools" >> "$GITHUB_PATH" + - name: Run release checks + run: npm run verify:release + - name: Upload browser failure evidence + if: failure() + uses: actions/upload-artifact@v4 + with: + name: browser-test-evidence + path: | + apps/web/test-results/ + apps/web/playwright-report/ + retention-days: 7 diff --git a/apps/web/app/fonts/bricolage-grotesque-latin-700.woff2 b/apps/web/app/fonts/bricolage-grotesque-latin-700.woff2 new file mode 100644 index 0000000..9e04f02 Binary files /dev/null and b/apps/web/app/fonts/bricolage-grotesque-latin-700.woff2 differ diff --git a/apps/web/app/fonts/figtree-latin-variable.woff2 b/apps/web/app/fonts/figtree-latin-variable.woff2 new file mode 100644 index 0000000..fcf1c90 Binary files /dev/null and b/apps/web/app/fonts/figtree-latin-variable.woff2 differ diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx index 47abeff..20fa138 100644 --- a/apps/web/app/layout.tsx +++ b/apps/web/app/layout.tsx @@ -1,9 +1,10 @@ import type { Metadata, Viewport } from "next"; -import { Bricolage_Grotesque, Figtree } from "next/font/google"; +import localFont from "next/font/local"; import "./globals.css"; -const display = Bricolage_Grotesque({ variable: "--font-bricolage", subsets: ["latin"], weight: ["700"] }); -const sans = Figtree({ variable: "--font-figtree", subsets: ["latin"] }); +// Keep production builds independent of Google Fonts availability. +const display = localFont({ src: "./fonts/bricolage-grotesque-latin-700.woff2", variable: "--font-bricolage", weight: "700", display: "swap" }); +const sans = localFont({ src: "./fonts/figtree-latin-variable.woff2", variable: "--font-figtree", weight: "300 900", display: "swap" }); export const metadata: Metadata = { title: "Quack", diff --git a/apps/web/lib/persist.ts b/apps/web/lib/persist.ts index c7d4a44..cd9bdda 100644 --- a/apps/web/lib/persist.ts +++ b/apps/web/lib/persist.ts @@ -21,18 +21,47 @@ function db(): UseStore { return store; } +const object = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); +const finite = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value); +const bounds = (value: unknown): value is Bounds => object(value) && + finite(value.minX) && finite(value.minY) && finite(value.maxX) && finite(value.maxY) && + value.minX <= value.maxX && value.minY <= value.maxY; + +// IndexedDB is an input boundary too: old app versions, failed imports and extensions can leave +// malformed values. A failed validation must preserve the stored value for recovery. +export function isSavedNotebook(value: unknown): value is SavedNotebook { + if (!object(value) || !Number.isSafeInteger(value.version) || (value.version as number) < 0 || !Array.isArray(value.strokes)) return false; + const ids = new Set(); + for (const stroke of value.strokes) { + if (!object(stroke) || typeof stroke.id !== "string" || !stroke.id || ids.has(stroke.id) || + typeof stroke.color !== "string" || !finite(stroke.width) || stroke.width < 0 || !finite(stroke.createdAt) || + !Array.isArray(stroke.points) || !stroke.points.length) return false; + ids.add(stroke.id); + for (const point of stroke.points) { + if (!object(point) || !finite(point.x) || !finite(point.y) || !finite(point.timestamp) || + (point.pressure !== undefined && (!finite(point.pressure) || point.pressure < 0 || point.pressure > 1))) return false; + } + } + if (value.dirty !== undefined && (!Array.isArray(value.dirty) || !value.dirty.every(bounds))) return false; + if (value.seen !== undefined && (!object(value.seen) || !Object.values(value.seen).every((item) => + object(item) && typeof item.dismissed === "boolean" && (item.area === null || bounds(item.area))))) return false; + return true; +} + // Failing to read is not the same as having nothing saved. Treating it as empty would let the // next stroke overwrite a student's notes. export async function loadNotebook(key: string = KEY): Promise { try { - const saved = await get(key, db()); - return saved ? { status: "found", saved } : { status: "empty" }; + const saved = await get(key, db()); + if (saved === undefined) return { status: "empty" }; + return isSavedNotebook(saved) ? { status: "found", saved } : { status: "failed" }; } catch { return { status: "failed" }; } } export async function saveNotebook(saved: SavedNotebook): Promise { + if (!isSavedNotebook(saved)) return false; try { await set(KEY, saved, db()); return true; @@ -45,12 +74,13 @@ export async function saveNotebook(saved: SavedNotebook): Promise { // other tab's work. So the write only goes ahead if the save is still the version we last knew, // checked in one transaction so that nothing can slip in between. `known` is null if there was none. export async function writeNotebook(saved: SavedNotebook, known: number | null, key: string = KEY): Promise { + if (!isSavedNotebook(saved) || (known !== null && (!Number.isSafeInteger(known) || known < 0 || saved.version < known))) return "failed"; let conflict = false; try { await update( key, (current) => { - conflict = current !== undefined && current.version !== known; + conflict = current === undefined ? known !== null : !isSavedNotebook(current) || current.version !== known; return conflict ? current! : saved; }, db(), diff --git a/apps/web/lib/scheduler.ts b/apps/web/lib/scheduler.ts index 6e076e2..5335e25 100644 --- a/apps/web/lib/scheduler.ts +++ b/apps/web/lib/scheduler.ts @@ -127,6 +127,7 @@ export class AnalysisScheduler { } checkNow() { + if (this.disposed) return; // Pressing again while a check runs can't make it faster, and used to start the same // expensive reading over and over on the server. const newest = this.newest; @@ -174,6 +175,7 @@ export class AnalysisScheduler { private mayStart(): boolean { const newest = this.newest; if (!newest) return true; + if (this.flights.some((flight) => flight.controller.signal.aborted)) return false; if (!this.deps.overlap) return false; if (this.flights.length >= MAX_IN_FLIGHT || newest.trigger !== "auto" || this.manualWaiting) return false; return this.deps.getVersion() > newest.capture.version; @@ -203,7 +205,7 @@ export class AnalysisScheduler { private cancelInFlight() { const flights = this.flights; - this.flights = []; + this.manualWaiting = false; for (const flight of flights) { flight.controller.abort(); flight.penUp?.(); @@ -239,6 +241,7 @@ export class AnalysisScheduler { } private async run(trigger: Trigger) { + if (this.disposed) return; const capture = this.deps.capture(trigger); if (!capture) { mark("check_skipped", undefined, { trigger, reason: "nothing_to_check" }); @@ -268,8 +271,14 @@ export class AnalysisScheduler { let answeredFor: Trigger | null = null; // Whether a newer check started while this one was under way. Its reply, or the newer one's, has the last word. const overtaken = () => this.flights.indexOf(flight) < this.flights.length - 1 || capture.version < this.delivered; + let onAbort: () => void = () => {}; try { - const result = await this.deps.send(capture, trigger, controller.signal); + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(new DOMException("aborted", "AbortError")); + controller.signal.addEventListener("abort", onAbort, { once: true }); + }); + const result = await Promise.race([this.deps.send(capture, trigger, controller.signal), aborted]); + if (controller.signal.aborted) throw new DOMException("aborted", "AbortError"); // The stroke under way isn't in the notebook yet, and could be this mistake's correction. // Pen-up comes just before the stroke is added, and this carries on after that. if (this.drawing) await new Promise((resolve) => (flight.penUp = resolve)); @@ -331,6 +340,7 @@ export class AnalysisScheduler { if (this.flights.length > 1 && (timedOut || !controller.signal.aborted)) this.deps.onBusyChange(true); } finally { clearTimeout(watchdog); + controller.signal.removeEventListener("abort", onAbort); const index = this.flights.indexOf(flight); if (index >= 0) this.flights.splice(index, 1); // Busy until the last check under way has finished, and said once. After a cancel, the list is already empty. diff --git a/apps/web/lib/types.ts b/apps/web/lib/types.ts index dcf9aef..7b12590 100644 --- a/apps/web/lib/types.ts +++ b/apps/web/lib/types.ts @@ -79,6 +79,7 @@ export type LineTrace = { }; export type AnalyzeResponse = { + // The server validates that alerts have new verified mistakes, unique ids, and no resolved overlap. snapshotVersion: number; requestId: string | null; status: AnalysisStatus; diff --git a/apps/web/package-lock.json b/apps/web/package-lock.json index 8645f74..47668ed 100644 --- a/apps/web/package-lock.json +++ b/apps/web/package-lock.json @@ -1252,6 +1252,13 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.1.0.tgz", + "integrity": "sha512-GPm/Dbvk5RLmhRnxXIa7nTpK0xtl4E0wj0MslsuNiJ1FIROU+2+9lgUC/dgF7Ey4QCWhfEsgxilMRMCdHPfDfw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -2590,16 +2597,17 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.5.tgz", + "integrity": "sha512-snPlVppFcYWfLLFQGhDxUyBcHNtMlWmn3jtj7pEaO+DP8pqL0IvAzXhMYmn1z9ASNtgmwLBdFxSf4tMUB0OTKA==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.8" + "@isaacs/nonrecursive-glob-match": "^1.1.0", + "brace-expansion": "^5.0.12" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" diff --git a/apps/web/package.json b/apps/web/package.json index cf63068..e13118d 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -6,10 +6,11 @@ "dev": "next dev", "build": "next build", "start": "next start", - "lint": "eslint", + "lint": "eslint --max-warnings=0", "test": "vitest run", "test:e2e": "playwright test", - "test:production": "playwright test --config playwright.production.config.ts" + "test:production": "playwright test --config playwright.production.config.ts", + "typecheck": "tsc --noEmit" }, "dependencies": { "@supabase/supabase-js": "2.118.0", @@ -33,5 +34,10 @@ "tailwindcss": "^4", "typescript": "^5", "vitest": "^5.0.5" + }, + "overrides": { + "@typescript-eslint/typescript-estree": { + "minimatch": "11.0.5" + } } } diff --git a/apps/web/playwright.config.ts b/apps/web/playwright.config.ts index dae225f..97f2abd 100644 --- a/apps/web/playwright.config.ts +++ b/apps/web/playwright.config.ts @@ -14,9 +14,11 @@ export default defineConfig({ expect: { timeout: 10_000 }, workers: 1, // the fake Meta server holds one script at a time fullyParallel: false, + forbidOnly: true, retries: 0, + outputDir: "test-results/journeys", reporter: [["list"], ["json", { outputFile: "test-results/results.json" }]], - use: { ...devices["Desktop Chrome"], viewport: { width: 1280, height: 800 } }, + use: { ...devices["Desktop Chrome"], viewport: { width: 1280, height: 800 }, trace: "retain-on-failure", screenshot: "only-on-failure" }, projects: [ // The notebook on its own, as it was before accounts, which is what proves that path hasn't moved. { name: "notebook", testIgnore: /(accounts|learning|lessons)\.spec/, use: { baseURL: `http://127.0.0.1:${port(3100)}` } }, diff --git a/apps/web/playwright.production.config.ts b/apps/web/playwright.production.config.ts index 6c52e87..ebdf3a1 100644 --- a/apps/web/playwright.production.config.ts +++ b/apps/web/playwright.production.config.ts @@ -7,9 +7,11 @@ const port = 3200 + Number(process.env.QUACK_E2E_PORT_SHIFT ?? 0); export default defineConfig({ testDir: "tests/production", workers: 1, + forbidOnly: true, retries: 0, - reporter: "list", - use: { ...devices["Desktop Chrome"], baseURL: `http://127.0.0.1:${port}` }, + outputDir: "test-results/production", + reporter: [["list"], ["json", { outputFile: "test-results/production-results.json" }]], + use: { ...devices["Desktop Chrome"], baseURL: `http://127.0.0.1:${port}`, trace: "retain-on-failure", screenshot: "only-on-failure" }, webServer: { command: `npx next start --port ${port} --hostname 127.0.0.1`, url: `http://127.0.0.1:${port}`, diff --git a/apps/web/tests/e2e/accounts.spec.ts b/apps/web/tests/e2e/accounts.spec.ts index 28a8372..457d0aa 100644 --- a/apps/web/tests/e2e/accounts.spec.ts +++ b/apps/web/tests/e2e/accounts.spec.ts @@ -212,6 +212,7 @@ test.describe("calibration", () => { await page.getByTestId("calibration-start").click(); await expect(page.getByTestId("calibration-next")).toBeDisabled(); await write(page, "1"); + await expect(page.getByTestId("calibration-next")).toBeEnabled(); await page.getByTestId("calibration-next").click(); await expect(page.getByTestId("calibration-problem")).toContainText("unfinished"); await expect(page.getByTestId("calibration-progress")).toHaveAttribute("aria-valuenow", "0"); diff --git a/apps/web/tests/production/safety.spec.ts b/apps/web/tests/production/safety.spec.ts new file mode 100644 index 0000000..62cc6a0 --- /dev/null +++ b/apps/web/tests/production/safety.spec.ts @@ -0,0 +1,114 @@ +import { expect, test } from "@playwright/test"; +import { strokeCount, write } from "../e2e/helpers"; + +test.beforeEach(async ({ page }) => { + await page.addInitScript(() => { + localStorage.setItem("quack.privacy-seen", "1"); + localStorage.setItem("quack.monitoring", "false"); + }); + // No backend or provider credentials: the compiled notebook handles unavailable checks. + await page.route("**/api/**", async (route) => { + const path = new URL(route.request().url()).pathname; + if (path === "/api/auth/config") return route.fulfill({ json: { mode: "off" } }); + if (path === "/api/health") return route.fulfill({ json: { status: "ok", modelConfigured: true, environment: "production" } }); + return route.fulfill({ status: 503, json: { detail: "unavailable" } }); + }); +}); + +test("production sends browser security headers", async ({ request }) => { + const response = await request.get("/"); + expect(response.ok()).toBe(true); + expect(response.headers()["x-content-type-options"]).toBe("nosniff"); + expect(response.headers()["x-frame-options"]).toBe("DENY"); + expect(response.headers()["referrer-policy"]).toBe("strict-origin-when-cross-origin"); + expect(response.headers()["permissions-policy"]).toContain("microphone=(self)"); + expect(response.headers()["x-powered-by"]).toBeUndefined(); +}); + +for (const viewport of [ + { width: 1280, height: 800 }, { width: 820, height: 1180 }, { width: 390, height: 844 }, +]) { + test(`compiled app loads without runtime errors at ${viewport.width}px`, async ({ page }) => { + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + await page.setViewportSize(viewport); + await page.goto("/"); + await expect(page.getByTestId("ink")).toBeVisible(); + await expect(page.getByTestId("tool-pen")).toBeVisible(); + expect(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth)).toBe(false); + expect(errors).toEqual([]); + }); +} + +test("loads both fonts from the app's own assets", async ({ page }) => { + const requests: string[] = []; + page.on("request", (request) => { + if (request.resourceType() === "font") requests.push(request.url()); + }); + await page.goto("/"); + await page.evaluate(async () => { await document.fonts.ready; }); + expect(requests.length).toBeGreaterThanOrEqual(2); + expect(requests.every((url) => new URL(url).origin === new URL(page.url()).origin)).toBe(true); + const loaded = await page.evaluate(() => { + const style = getComputedStyle(document.documentElement); + return ["--font-bricolage", "--font-figtree"].map((name) => { + const family = style.getPropertyValue(name).trim().split(",")[0]; + return Boolean(family) && document.fonts.check(`700 16px ${family}`); + }); + }); + expect(loaded).toEqual([true, true]); +}); + +test("a failed check keeps ink saved across reload", async ({ page }) => { + await page.goto("/"); + await expect(page.getByTestId("ink")).toBeVisible(); + await write(page, "2*3=5"); + const count = await strokeCount(page); + expect(count).toBeGreaterThan(0); + await page.getByTestId("check").click(); + await expect(page.getByText("Quack can't reach its server. Your notes are safe.")).toBeVisible(); + // Wait for the actual transaction, rather than assuming the debounce has fired. + await expect.poll(() => page.evaluate(() => new Promise((resolve, reject) => { + const request = indexedDB.open("quack"); + request.onerror = () => reject(request.error); + request.onsuccess = () => { + const db = request.result; + const read = db.transaction("notes").objectStore("notes").get("notebook"); + read.onsuccess = () => { resolve(read.result?.strokes?.length ?? 0); db.close(); }; + read.onerror = () => { reject(read.error); db.close(); }; + }; + }))).toBe(count); + await page.reload(); + await expect.poll(() => strokeCount(page)).toBe(count); +}); + +test("corrupted saved notes produce a visible recovery message and remain intact", async ({ page }) => { + await page.goto("/"); + await expect(page.getByTestId("ink")).toBeVisible(); + await page.evaluate(() => new Promise((resolve, reject) => { + const request = indexedDB.open("quack"); + request.onsuccess = () => { + const db = request.result; + const tx = db.transaction("notes", "readwrite"); + tx.objectStore("notes").put({ version: 1, strokes: "recoverable-original" }, "notebook"); + tx.oncomplete = () => { db.close(); resolve(); }; + tx.onerror = () => { db.close(); reject(tx.error); }; + }; + request.onerror = () => reject(request.error); + })); + await page.reload(); + await expect(page.getByText("Quack couldn't open your saved notes, so new writing won't be saved.")).toBeVisible(); + await write(page, "2x"); + await page.waitForTimeout(700); // past the normal save debounce + const stored = await page.evaluate(() => new Promise((resolve, reject) => { + const request = indexedDB.open("quack"); + request.onsuccess = () => { + const db = request.result; + const read = db.transaction("notes").objectStore("notes").get("notebook"); + read.onsuccess = () => { db.close(); resolve(read.result); }; + read.onerror = () => { db.close(); reject(read.error); }; + }; + request.onerror = () => reject(request.error); + })); + expect(stored).toEqual({ version: 1, strokes: "recoverable-original" }); +}); diff --git a/apps/web/tests/unit/release-safety.test.ts b/apps/web/tests/unit/release-safety.test.ts new file mode 100644 index 0000000..a4f2268 --- /dev/null +++ b/apps/web/tests/unit/release-safety.test.ts @@ -0,0 +1,228 @@ +import "fake-indexeddb/auto"; +import { createStore, del, get, set } from "idb-keyval"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { analyze, ApiError, sendJson, setAccount, setStudySession } from "@/lib/api"; +import { isSavedNotebook, loadNotebook, writeNotebook, type SavedNotebook } from "@/lib/persist"; +import { AnalysisScheduler, RequestTimedOut } from "@/lib/scheduler"; +import { NotebookStore } from "@/lib/store"; +import type { AnalyzeResponse, PenStroke } from "@/lib/types"; + +const ink: PenStroke = { + id: "s1", color: "#000", width: 2, createdAt: 1, + points: [{ x: 1, y: 2, timestamp: 1, pressure: .5 }], +}; +const valid = (): SavedNotebook => ({ strokes: [structuredClone(ink)], version: 1 }); +const db = createStore("quack", "notes"); +let sequence = 0; +const key = () => `release-safety-${sequence++}`; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + vi.useRealTimers(); + setAccount(async () => null, () => {}); + setStudySession(() => null); +}); + +describe("saved notebook integrity", () => { + const invalid: [string, unknown][] = [ + ["null", null], ["non-object", []], ["missing version", { strokes: [] }], + ["negative version", { ...valid(), version: -1 }], + ["fractional version", { ...valid(), version: 1.5 }], + ["unsafe version", { ...valid(), version: Number.MAX_SAFE_INTEGER + 1 }], + ["invalid strokes", { version: 1, strokes: [null] }], + ["empty stroke", { version: 1, strokes: [{ ...ink, points: [] }] }], + ["duplicate ids", { version: 1, strokes: [ink, ink] }], + ["invalid coordinates", { version: 1, strokes: [{ ...ink, points: [{ x: NaN, y: 1, timestamp: 0 }] }] }], + ["infinite width", { version: 1, strokes: [{ ...ink, width: Infinity }] }], + ["negative width", { version: 1, strokes: [{ ...ink, width: -1 }] }], + ["missing timestamp", { version: 1, strokes: [{ ...ink, points: [{ x: 1, y: 1 }] }] }], + ["invalid pressure", { version: 1, strokes: [{ ...ink, points: [{ x: 1, y: 1, timestamp: 1, pressure: 2 }] }] }], + ["reversed bounds", { ...valid(), dirty: [{ minX: 10, maxX: 0, minY: 0, maxY: 10 }] }], + ["invalid seen record", { ...valid(), seen: { m: { dismissed: "yes", area: null } } }], + ]; + + it.each(invalid)("preserves a corrupt value: %s", async (_label, stored) => { + const name = key(); + await set(name, stored, db); + expect(isSavedNotebook(stored)).toBe(false); + expect(await loadNotebook(name)).toEqual({ status: "failed" }); + expect(await writeNotebook(valid(), null, name)).toBe("conflict"); + expect(await get(name, db)).toEqual(stored); + }); + + it("accepts older notebooks without optional metadata", async () => { + const name = key(); + await set(name, valid(), db); + expect(await loadNotebook(name)).toEqual({ status: "found", saved: valid() }); + }); + + it("reports a genuinely absent save as empty", async () => { + expect(await loadNotebook(key())).toEqual({ status: "empty" }); + }); + + it("does not resurrect notes deleted by another tab", async () => { + const name = key(); + await set(name, valid(), db); + await del(name, db); + expect(await writeNotebook({ ...valid(), version: 2 }, 1, name)).toBe("conflict"); + expect(await get(name, db)).toBeUndefined(); + }); + + it("allows exactly one of two concurrent first writers", async () => { + const name = key(); + const results = await Promise.all([ + writeNotebook(valid(), null, name), + writeNotebook({ strokes: [], version: 2 }, null, name), + ]); + expect(results.sort()).toEqual(["conflict", "saved"]); + }); + + it("rejects invalid writes before they replace good notes", async () => { + const name = key(); + await set(name, valid(), db); + expect(await writeNotebook({ ...valid(), version: NaN }, 1, name)).toBe("failed"); + expect(await writeNotebook({ ...valid(), version: 0 }, 1, name)).toBe("failed"); + expect(await get(name, db)).toEqual(valid()); + }); + + it("keeps saving disabled when hydration found corruption", async () => { + const name = key(); + const corrupted = { strokes: "recover this", version: 1 }; + await set(name, corrupted, db); + const notebook = new NotebookStore({ key: name }); + expect((await notebook.hydrate()).status).toBe("failed"); + notebook.add(ink); + notebook.flush(); + expect(await get(name, db)).toEqual(corrupted); + }); +}); + +describe("HTTP failures and account isolation", () => { + it.each([400, 401, 403, 413, 422, 429, 500, 503])("rejects HTTP %i without accepting its body", async (status) => { + const signedOut = vi.fn(); + setAccount(async () => "token", signedOut); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("private server error", { status }))); + await expect(sendJson("GET", "/api/test")).rejects.toMatchObject({ status }); + expect(signedOut).toHaveBeenCalledTimes(status === 401 ? 1 : 0); + }); + + it("passes the abort signal and signed-in token to fetch", async () => { + const controller = new AbortController(); + const fetchMock = vi.fn().mockResolvedValue(new Response('{"ok":true}')); + vi.stubGlobal("fetch", fetchMock); + setAccount(async () => "test-token", () => {}); + await sendJson("POST", "/api/test", { value: 1 }, controller.signal); + expect(fetchMock).toHaveBeenCalledWith("/api/test", expect.objectContaining({ + signal: controller.signal, body: '{"value":1}', + headers: { authorization: "Bearer test-token", "content-type": "application/json" }, + })); + }); + + it("does not send an old token after sign-out", async () => { + const fetchMock = vi.fn().mockImplementation(async () => new Response("{}")); + vi.stubGlobal("fetch", fetchMock); + setAccount(async () => "old-token", () => {}); + await sendJson("GET", "/api/test"); + setAccount(async () => null, () => {}); + await sendJson("GET", "/api/test"); + expect(fetchMock.mock.calls[1][1].headers).toEqual({}); + }); + + it("includes the current study session in analysis requests", async () => { + const fetchMock = vi.fn().mockResolvedValue(new Response("{}")); + vi.stubGlobal("fetch", fetchMock); + setStudySession(() => "study-1"); + await analyze("ink-session", 3, "data:image/png;base64,", "manual", ["m1"]); + expect(JSON.parse(fetchMock.mock.calls[0][1].body)).toMatchObject({ studySessionId: "study-1", snapshotVersion: 3 }); + }); + + it("returns a typed error without echoing sensitive response content", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("secret", { status: 500 }))); + await expect(sendJson("GET", "/api/test")).rejects.toBeInstanceOf(ApiError); + await expect(sendJson("GET", "/api/test")).rejects.not.toThrow("secret"); + }); +}); + +describe("scheduler cancellation and lifecycle", () => { + function setup() { + vi.useFakeTimers(); + const deliveries: ((response: AnalyzeResponse) => void)[] = []; + const send = vi.fn(() => + new Promise((resolve) => deliveries.push(resolve))); + const restore = vi.fn(); + const onResult = vi.fn(); + const onError = vi.fn(); + const onBusyChange = vi.fn(); + const scheduler = new AnalysisScheduler({ + getVersion: () => 1, + capture: () => ({ hash: "ink", version: 1, image: "data:", restore }), + send, onResult, onError, onBusyChange, requestTimeoutMs: 100, + }); + const result = { snapshotVersion: 1, stale: false, modelIssue: null } as AnalyzeResponse; + return { scheduler, send, restore, onResult, onError, onBusyChange, deliveries, result }; + } + + it("times out even if the transport ignores abort, then retries", async () => { + const h = setup(); + h.scheduler.checkNow(); + await vi.advanceTimersByTimeAsync(100); + expect(h.onError).toHaveBeenCalledWith(expect.any(RequestTimedOut), "manual"); + expect(h.restore).toHaveBeenCalledOnce(); + expect(h.onBusyChange).toHaveBeenLastCalledWith(false); + await vi.advanceTimersByTimeAsync(2000); + expect(h.send).toHaveBeenCalledTimes(2); + h.scheduler.dispose(); + await vi.advanceTimersByTimeAsync(0); + }); + + it("does not apply a late response after disposal", async () => { + const h = setup(); + h.scheduler.checkNow(); + h.scheduler.dispose(); + h.deliveries[0](h.result); + await vi.advanceTimersByTimeAsync(10_000); + expect(h.onResult).not.toHaveBeenCalled(); + expect(h.onError).not.toHaveBeenCalled(); + expect(h.send).toHaveBeenCalledOnce(); + expect(h.onBusyChange).toHaveBeenLastCalledWith(false); + }); + + it("cannot be restarted after disposal", async () => { + const h = setup(); + h.scheduler.dispose(); + h.scheduler.checkNow(); + h.scheduler.notifyEdit(); + h.scheduler.setEnabled(true); + await vi.advanceTimersByTimeAsync(10_000); + expect(h.send).not.toHaveBeenCalled(); + }); + + it("turning monitoring off clears a queued manual request", async () => { + const h = setup(); + h.scheduler.notifyEdit(); + await vi.advanceTimersByTimeAsync(800); + h.scheduler.checkNow(); + h.scheduler.setEnabled(false); + await vi.advanceTimersByTimeAsync(10_000); + expect(h.send).toHaveBeenCalledOnce(); + expect(h.onError).not.toHaveBeenCalled(); + h.scheduler.dispose(); + }); + + it("re-enabling immediately after cancellation starts one new run", async () => { + const h = setup(); + h.scheduler.checkNow(); + h.scheduler.setEnabled(false); + h.scheduler.setEnabled(true); + await vi.advanceTimersByTimeAsync(0); + expect(h.send).toHaveBeenCalledTimes(2); + h.deliveries[0](h.result); + await vi.advanceTimersByTimeAsync(0); + expect(h.onResult).not.toHaveBeenCalled(); + h.deliveries[1](h.result); + await vi.advanceTimersByTimeAsync(0); + expect(h.onResult).toHaveBeenCalledOnce(); + h.scheduler.dispose(); + }); +}); diff --git a/apps/web/vitest.config.mts b/apps/web/vitest.config.mts index 62ddde1..954a34d 100644 --- a/apps/web/vitest.config.mts +++ b/apps/web/vitest.config.mts @@ -3,5 +3,5 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ resolve: { alias: { "@": fileURLToPath(new URL(".", import.meta.url)) } }, - test: { environment: "jsdom", include: ["tests/unit/**/*.test.ts"] }, + test: { allowOnly: false, environment: "jsdom", include: ["tests/unit/**/*.test.ts"] }, }); diff --git a/docs/RELEASE_SAFETY.md b/docs/RELEASE_SAFETY.md new file mode 100644 index 0000000..440fa86 --- /dev/null +++ b/docs/RELEASE_SAFETY.md @@ -0,0 +1,121 @@ +# Release verification and safeguards + +## Reproduce the checks + +Use Python 3.12, Node 22, PostgreSQL 18 and Gitleaks 8.30.1. From a clean checkout: + +```bash +python3.12 -m venv services/api/.venv +services/api/.venv/bin/python -m pip install -r services/api/requirements.lock -r services/api/requirements-tooling.lock -r services/api/requirements-dev.txt +npm ci --prefix apps/web +(cd apps/web && npx playwright install --with-deps chromium) +# Create an empty LOCAL disposable database named quack_ledger_test. +export QUACK_LEDGER_TEST_DATABASE_URL=postgresql://localhost/quack_ledger_test +npm run verify:release +``` + +The database role needs permission to create a test role for the row-security regression. +Ledger tests truncate only their disposable local database. Never supply production credentials. +`QUACK_TEST_PYTHON` and `QUACK_GITLEAKS` can select explicit tooling paths. +`QUACK_E2E_PORT_SHIFT` shifts the browser suite's ports if other development servers are running. + +The command checks dependency consistency, locked inputs, focused tests, the public-file allowlist, +reachable publication history, staged and historical secrets, Ruff, mypy, offline backend tests, +ESLint, TypeScript, frontend units, the production build, compiled browser checks and all existing +browser journeys. It refuses skipped backend tests and unknown pytest settings or markers. +The two paid live tests are deliberately deselected. No model credits are needed. +Release browser verification enables the existing performance and screenshot journeys as well. +JSON reports must contain passing cases and zero skipped, failed or flaky cases. Compiled and +development-server evidence uses separate output directories so the latter cannot erase the former. +Historical prompt comparisons use a classification-only fixture checked against the published +synthetic dataset. It contains no raw model text, usage records, images or private evaluation paths. + +Production credentials and API routing overrides are removed from the verification subprocess +environment. Browser journeys use isolated temporary storage, local account stand-ins and a fake +model endpoint. The compiled suite stubs API responses inside the browser. Production debug-tool +restrictions remain covered by the existing compiled tests. + +Run advisory checks separately (they need advisory-service network access): + +```bash +services/api/.venv/bin/python -m pip_audit --no-deps --disable-pip -r services/api/requirements.lock -r services/api/requirements-tooling.lock +npm audit --prefix apps/web +``` + +The workflow runs these audits and the complete release command on pull requests and pushes to +`main`, plus manual dispatch. One GitHub-hosted job avoids duplicate branch/PR browser runs and +uses an ephemeral PostgreSQL service. Chromium is installed explicitly; browser traces and +screenshots are retained for seven days after failures. Focused Playwright and Vitest tests fail +locally and in CI. An additional source guard rejects committed `test.only` forms. + +Local evidence does not establish hosted CI or deployment health. Check the workflow attached +to the exact reviewed commit before merging. Advisory results are point-in-time observations. + +## Accounting: production versus local development + +Production uses **PostgreSQL-backed accounting**. Before a model operation, an advisory transaction +lock coordinates independent connections while allocating a durable debit for the complete retry +allowance. The router's attempts share that debit; they do not multiply it again. Account scope +ignores rotating client session ids. Project, account, request and hourly ceilings remain enforced. +Successful operations settle only when every usage receipt is complete and successful. Unknown +usage, timeout, failed retry, cancellation and crash retain the conservative reservation indefinitely. +Cancelled requests also write a usage receipt. Database failures prevent further model operations; +invalid receipts retain the debit and stop subsequent spending. No production spending records +are deleted or reset by verification or deployment. + +Local development can use the append-only JSON file ledger. Each retry independently checks +committed spend and reserves its next worst-case cost. Replay validates every incoming record +before adding any of them. Invalid costs, timestamps or session ids fail closed. Write failures +count completed usage in memory and block further spending for that process. Hourly accounting +handles timestamps received out of order. These disk and replay safeguards apply to the file +backend, not to the PostgreSQL implementation. File reservations coordinate only one process; +file accounting must not replace the production transactional ledger. + +Retain one production Modal container: notebook analysis sessions, rate windows and other state +remain process-local even though production spending is coordinated by PostgreSQL. Existing +inference quotas are deployment configuration, not test defaults. Do not increase them for a +release. Provider cost estimates include a fixed input allowance and are not provider billing caps. + +## Student-facing safety + +Handwriting recognition proposes text; the restricted parser and deterministic SymPy verifier +establish mathematical evidence. Runtime response validation rejects unverified or duplicate +mistakes, inconsistent selected mistakes, present/resolved overlap, and alerts without a current +new verified mistake. These checks run with Python assertions disabled and do not turn model +confidence into mathematical proof. + +Notebook hydration validates versions, unique stroke ids, finite geometry, points and metadata. +Unreadable or corrupt values remain stored and new writing cannot overwrite them. Cross-tab +version transactions prevent simultaneous first writers and recreation after another tab deletes +notes. Export, deletion and per-account storage separation retain their existing behavior. + +The production scheduler still supports checks of newer ink while an older check runs. Aborted +transports cannot deliver late warnings; a watchdog releases checks even if the transport ignores +abort. Cancelled captures finish cleanup before replacement work starts. Existing stale-reply, +account, mastery and practice regressions cover the broader interaction. + +Bricolage Grotesque and Figtree are bundled as Latin WOFF2 assets with the existing +[SIL font licenses](licenses/THIRD_PARTY_NOTICES.txt). Their names, weights and CSS variables are +preserved. Builds no longer fetch Google Fonts; compiled browser tests verify same-origin font +requests, loaded faces, responsive rendering and saved-ink recovery. + +## HTTP diagnostics + +Every response includes a fresh server-generated `X-Request-ID`. Configured CORS origins can read +it. Nested application logs carry the same context. HTTP events contain the route template, +method, status and duration, without literal student paths, query strings, request bodies or auth +headers. Unhandled failures return a generic response; logs contain the exception class and code +locations, not exception messages, source code, handwriting or local variables. API responses use +`Cache-Control: no-store`. The correlation id is independent of the analysis payload's `requestId`. + +The Next.js rewrite forwards backend response headers through the same-origin frontend proxy. +Vercel and Modal generate their own platform request identifiers; those are separate from Quack's. + +## Limits of the evidence + +Synthetic and mocked checks do not establish real handwriting accuracy, physical Apple Pencil or +iPad behavior, production Supabase registration/deletion/recovery flows, personalized recognition +quality, or provider billing. A release must independently inspect actual hosted CI, frontend +and backend source versions, HTTPS, health/readiness, auth guards, proxy behavior and debug-route +restrictions. Deploying the frontend does not deploy Modal. A live paid inference test requires +explicit release-specific authorization. Never exercise destructive database tests in production. diff --git a/package.json b/package.json new file mode 100644 index 0000000..eae8a24 --- /dev/null +++ b/package.json @@ -0,0 +1,8 @@ +{ + "name": "quack", + "private": true, + "scripts": { + "verify": "python3 scripts/verify.py", + "verify:release": "python3 scripts/verify.py --build --browser" + } +} diff --git a/scripts/check_browser_results.py b/scripts/check_browser_results.py new file mode 100755 index 0000000..4e53b40 --- /dev/null +++ b/scripts/check_browser_results.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Fail release verification if Playwright omitted cases or reported retries/failures.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + + +def main() -> int: + if len(sys.argv) != 2: + raise SystemExit("usage: check_browser_results.py REPORT.json") + report = json.loads(Path(sys.argv[1]).read_text()) + stats = report["stats"] + if report.get("errors") or stats["expected"] <= 0 or any(stats[key] for key in ("skipped", "unexpected", "flaky")): + print(f"Incomplete browser verification: {stats}") + return 1 + print(f"Browser completeness: {stats['expected']} passed, zero skipped, failed or flaky") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_publication.py b/scripts/check_publication.py index af461f0..f2b5526 100755 --- a/scripts/check_publication.py +++ b/scripts/check_publication.py @@ -19,6 +19,7 @@ "docs/ARCHITECTURE.md", "docs/SETUP.md", "docs/TESTING.md", + "docs/RELEASE_SAFETY.md", "docs/PUBLICATION.md", "docs/CREDITS.md", "docs/hackathon/JUDGE_GUIDE.md", diff --git a/scripts/check_release.py b/scripts/check_release.py new file mode 100644 index 0000000..67c3e36 --- /dev/null +++ b/scripts/check_release.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Validate locked release inputs, published files and committed test selection.""" + +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def main() -> int: + errors: list[str] = [] + names = subprocess.check_output(["git", "ls-files", "-z"], cwd=ROOT).decode().split("\0") + for name in filter(None, names): + path = ROOT / name + if not path.is_file(): + errors.append(f"Missing published file: {name}") + continue + if ( + "/tests/" in name and path.suffix in {".ts", ".tsx", ".py"} + and re.search(r"\b(?:test|it|describe)\s*\.\s*only\s*[(<]", path.read_text()) + ): + errors.append(f"Focused test committed: {name}") + package = json.loads((ROOT / "apps/web/package.json").read_text()) + locked = json.loads((ROOT / "apps/web/package-lock.json").read_text())["packages"][""] + for group in ("dependencies", "devDependencies"): + if package[group] != locked[group]: + errors.append(f"Web {group} differ from lockfile") + for name in ("requirements.lock", "requirements-tooling.lock"): + for line in (ROOT / "services/api" / name).read_text().splitlines(): + if line.strip() and not line.startswith("#") and not re.fullmatch(r"[\w.-]+==[\w.+-]+", line): + errors.append(f"Unpinned Python input: {name}") + if "next/font/google" in (ROOT / "apps/web/app/layout.tsx").read_text(): + errors.append("Build must use bundled fonts") + for error in errors: + print(error) + print(f"Release inputs: {len(errors)} errors") + return bool(errors) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/public-files.txt b/scripts/public-files.txt index 10cc22e..dfe2a9f 100644 --- a/scripts/public-files.txt +++ b/scripts/public-files.txt @@ -427,3 +427,17 @@ docs/media/hint-detail.webp docs/media/correction-detail.webp docs/media/practice-detail.webp docs/media/learning-detail.webp +.github/workflows/verify.yml +apps/web/app/fonts/bricolage-grotesque-latin-700.woff2 +apps/web/app/fonts/figtree-latin-variable.woff2 +apps/web/tests/production/safety.spec.ts +apps/web/tests/unit/release-safety.test.ts +docs/RELEASE_SAFETY.md +package.json +scripts/check_release.py +scripts/verify.py +services/api/quack/observability.py +services/api/requirements-tooling.lock +services/api/tests/test_release_safety.py +services/api/tests/fixtures/recorded-routing.json +scripts/check_browser_results.py diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 0000000..87e5a4c --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Run Quack's release checks. No live model calls, deployments or production credentials.""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def test_environment() -> dict[str, str]: + """Never inherit production keys, databases, routes or budgets into release checks.""" + keep = {"QUACK_TEST_PYTHON", "QUACK_LEDGER_TEST_DATABASE_URL", "QUACK_E2E_PORT_SHIFT", "QUACK_GITLEAKS"} + env = { + key: value for key, value in os.environ.items() + if not key.startswith(("QUACK_", "MODEL_", "SUPABASE_", "NEXT_PUBLIC_")) or key in keep + } + env.pop("API_URL", None) + env.update(CI="true", QUACK_ENV="production", MODEL_API_KEY="", + QUACK_RELEASE_VERIFY="1", NEXT_TELEMETRY_DISABLED="1") + return env + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--browser", action="store_true", help="also run all mocked Playwright journeys") + parser.add_argument("--build", action="store_true", help="also compile the production web app") + args = parser.parse_args() + env = test_environment() + if args.browser: + # Existing visual/performance journeys are opt-in during development, required for release. + env.update(QUACK_PERF="1", QUACK_SCREENSHOTS="1") + if not env.get("QUACK_LEDGER_TEST_DATABASE_URL"): + parser.error("set QUACK_LEDGER_TEST_DATABASE_URL to a disposable local quack_ledger_test database") + gitleaks = env.get("QUACK_GITLEAKS") or shutil.which("gitleaks") + if not gitleaks: + parser.error("install Gitleaks 8.30.1 or set QUACK_GITLEAKS to its executable") + local_python = ROOT / "services/api/.venv/bin/python" + python = os.environ.get("QUACK_TEST_PYTHON") or (str(local_python) if local_python.exists() else sys.executable) + checks = [ + ("Dependency consistency", ".", [python, "-m", "pip", "check"]), + ("Release inputs and focused-test guard", ".", [python, "scripts/check_release.py"]), + ("Public file and history policy", ".", [python, "scripts/check_publication.py", "--history"]), + ("History secret scan", ".", [gitleaks, "git", "--redact", "--no-banner", "--log-opts=--all"]), + ("Staged secret scan", ".", [gitleaks, "git", "--redact", "--no-banner", "--pre-commit", "--staged"]), + ("Python lint", "services/api", [python, "-m", "ruff", "check", "quack", "tests"]), + ("Python types", "services/api", [python, "-m", "mypy", "quack"]), + ("Backend regressions", "services/api", [python, "-m", "pytest", "-q", "-m", "not live"]), + ("Web lint", "apps/web", ["npm", "run", "lint"]), + ("Web types", "apps/web", ["npm", "run", "typecheck"]), + ("Web logic", "apps/web", ["npm", "test"]), + ] + if args.build: + checks.append(("Production build", "apps/web", ["npm", "run", "build"])) + if args.browser: + checks.append(("Compiled app safety", "apps/web", ["npm", "run", "test:production"])) + checks.append(("Compiled browser completeness", ".", [python, "scripts/check_browser_results.py", + "apps/web/test-results/production-results.json"])) + if args.browser: + checks.append(("Browser journeys (fake model)", "apps/web", ["npm", "run", "test:e2e"])) + checks.append(("Browser journey completeness", ".", [python, "scripts/check_browser_results.py", + "apps/web/test-results/results.json"])) + for label, directory, command in checks: + print(f"\n{label}", flush=True) + result = subprocess.run(command, cwd=ROOT / directory, env=env, check=False) + if result.returncode: + print(f"\nFAILED: {label} (exit {result.returncode})", file=sys.stderr) + return result.returncode + print(f"\nPassed all {len(checks)} checks.", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/api/pyproject.toml b/services/api/pyproject.toml index 5362729..a835831 100644 --- a/services/api/pyproject.toml +++ b/services/api/pyproject.toml @@ -9,7 +9,7 @@ pythonpath = ["."] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" markers = ["live: calls the real Meta Model API and spends credits; needs MODEL_API_KEY"] -addopts = "-m 'not live'" +addopts = "-m 'not live' --strict-markers --strict-config" [tool.ruff] line-length = 125 diff --git a/services/api/quack/account_api.py b/services/api/quack/account_api.py index ebcab23..7df771c 100644 --- a/services/api/quack/account_api.py +++ b/services/api/quack/account_api.py @@ -43,6 +43,7 @@ PracticeReveal, PracticeStart, PracticeSummary, + PracticeTopic, PracticeTopicOption, Profile, Recap, @@ -218,14 +219,16 @@ def _owned(store: UserStore, user_id: str, session_id: str | None) -> str | None return session_id if session_id and store.session(user_id, session_id) else None -def _general_topic(store: UserStore, user_id: str, origin: str | None) -> str: +def _general_topic(store: UserStore, user_id: str, origin: str | None) -> PracticeTopic: """What the recap recommended, if it did, and otherwise simply the first one on the list.""" session = store.session(user_id, origin) if origin else None if session and session["recap_json"]: recommendation = Recap.model_validate_json(session["recap_json"]).recommendation if recommendation: - return recommendation.topic + for topic in practice_service.SUPPORTED: + if topic == recommendation.topic: + return topic return practice_service.SUPPORTED[0] @@ -498,7 +501,11 @@ async def practice_topics() -> PracticeOptions: @router.post("/practice/sessions") async def start_practice(body: PracticeStart, identity: Identity = you) -> PracticeProblem: user_id = identity.user_id - topic = body.topic or await asyncio.to_thread(_general_topic, store, user_id, body.origin_session_id) + topic: PracticeTopic + if body.topic is not None: + topic = body.topic + else: + topic = await asyncio.to_thread(_general_topic, store, user_id, body.origin_session_id) origin = await asyncio.to_thread(_owned, store, user_id, body.origin_session_id) session = await asyncio.to_thread(store.new_practice_session, user_id, origin, topic) # Only one of the student's own. Anything else simply isn't linked. diff --git a/services/api/quack/config.py b/services/api/quack/config.py index e302ac1..f74a4f3 100644 --- a/services/api/quack/config.py +++ b/services/api/quack/config.py @@ -15,18 +15,19 @@ class Settings(BaseSettings): model_config = SettingsConfigDict( - env_file=(REPO_ROOT / ".env", API_ROOT / ".env"), extra="ignore", populate_by_name=True + env_file=(REPO_ROOT / ".env", API_ROOT / ".env"), extra="ignore", populate_by_name=True, + allow_inf_nan=False, ) model_api_key: str = Field(default="", alias="MODEL_API_KEY", repr=False) model_api_base_url: str = Field(default="https://api.meta.ai/v1", alias="MODEL_API_BASE_URL") # Development-only routes stay hidden unless this is set on purpose. - env: str = Field(default="production", alias="QUACK_ENV") + env: Literal["production", "development"] = Field(default="production", alias="QUACK_ENV") cors_origins: str = Field(default="http://localhost:3000", alias="QUACK_CORS_ORIGINS") - rate_limit_per_minute: int = Field(default=60, alias="QUACK_RATE_LIMIT_PER_MINUTE") + rate_limit_per_minute: int = Field(default=60, gt=0, alias="QUACK_RATE_LIMIT_PER_MINUTE") # Client addresses can be forged through a proxy, so this cap is what actually bounds spend rate. - global_rate_limit_per_minute: int = Field(default=300, alias="QUACK_GLOBAL_RATE_LIMIT_PER_MINUTE") + global_rate_limit_per_minute: int = Field(default=300, gt=0, alias="QUACK_GLOBAL_RATE_LIMIT_PER_MINUTE") # JSON object of per-purpose overrides, for example: # {"transcribe": {"model": "muse-wonder-1.0", "reasoning_effort": "medium"}} @@ -40,21 +41,21 @@ class Settings(BaseSettings): confirm_in_parallel: bool = Field(default=True, alias="QUACK_CONFIRM_IN_PARALLEL") # How long the second reading can take before another goes alongside it. Most finish in 6 s, and # those that use up all their tokens take 19 s. See `AnalysisService._second_reading`. - second_reading_patience_s: float = Field(default=14.0, alias="QUACK_SECOND_READING_PATIENCE_S") + second_reading_patience_s: float = Field(default=14.0, gt=0, alias="QUACK_SECOND_READING_PATIENCE_S") # Re-read when the fast reading was unclear. Off until evaluation shows it pays for itself. second_reading_on_uncertain: bool = Field(default=False, alias="QUACK_SECOND_READING_ON_UNCERTAIN") - reading_cache_size: int = Field(default=256, alias="QUACK_READING_CACHE_SIZE") + reading_cache_size: int = Field(default=256, ge=0, alias="QUACK_READING_CACHE_SIZE") # Lines read and agreed on, across sessions, so unchanged lines needn't be read again. - line_memory_size: int = Field(default=20_000, alias="QUACK_LINE_MEMORY_SIZE") - verify_timeout_s: float = Field(default=4.0, alias="QUACK_VERIFY_TIMEOUT_S") + line_memory_size: int = Field(default=20_000, ge=0, alias="QUACK_LINE_MEMORY_SIZE") + verify_timeout_s: float = Field(default=4.0, gt=0, alias="QUACK_VERIFY_TIMEOUT_S") # Credit guard rails. The project ceiling leaves US$15 of the US$150 allocation in reserve. - budget_project_usd: float = Field(default=135.0, alias="QUACK_BUDGET_PROJECT_USD") - budget_session_usd: float = Field(default=2.0, alias="QUACK_BUDGET_SESSION_USD") - budget_request_usd: float = Field(default=0.50, alias="QUACK_BUDGET_REQUEST_USD") + budget_project_usd: float = Field(default=135.0, ge=0, alias="QUACK_BUDGET_PROJECT_USD") + budget_session_usd: float = Field(default=2.0, ge=0, alias="QUACK_BUDGET_SESSION_USD") + budget_request_usd: float = Field(default=0.50, ge=0, alias="QUACK_BUDGET_REQUEST_USD") # A session id is chosen by the client, so per-session limits only stop a runaway tab. # This one stops anyone draining the credits quickly, whatever ids they use. - budget_hourly_usd: float = Field(default=5.0, alias="QUACK_BUDGET_HOURLY_USD") + budget_hourly_usd: float = Field(default=5.0, ge=0, alias="QUACK_BUDGET_HOURLY_USD") ledger_path: Path = Field(default=API_ROOT / "var" / "usage.jsonl", alias="QUACK_LEDGER_PATH") ledger_backend: Literal["file", "postgres"] = Field(default="file", alias="QUACK_LEDGER_BACKEND") diff --git a/services/api/quack/llm/ledger.py b/services/api/quack/llm/ledger.py index c14f5f4..145a45a 100644 --- a/services/api/quack/llm/ledger.py +++ b/services/api/quack/llm/ledger.py @@ -3,14 +3,34 @@ from __future__ import annotations import json +import logging +import math import threading import time from collections import defaultdict, deque from collections.abc import Iterator -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager, nullcontext from dataclasses import asdict, dataclass, field from pathlib import Path +log = logging.getLogger("quack.ledger") + + +def _nonnegative(value: object) -> float: + if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value) or value < 0: + raise ValueError("ledger amounts must be finite, nonnegative numbers") + return float(value) + + +def _validated(record: object) -> dict: + if not isinstance(record, dict): + raise ValueError("ledger record must be an object") + _nonnegative(record.get("estimated_usd", 0.0)) + _nonnegative(record.get("timestamp", time.time())) + if not isinstance(record.get("session_id", ""), str): + raise ValueError("ledger session must be text") + return record + class BudgetExceeded(Exception): def __init__(self, scope: str, spent: float, limit: float) -> None: @@ -51,6 +71,9 @@ def __init__( hourly_limit: float | None = None, ) -> None: self.path = path + for limit in (project_limit, session_limit, request_limit, hourly_limit): + if limit is not None: + _nonnegative(limit) self.project_limit = project_limit self.session_limit = session_limit self.request_limit = request_limit @@ -63,6 +86,7 @@ def __init__( self._reserved_project = 0.0 self._reserved_session: dict[str, float] = defaultdict(float) self._read_up_to = 0 + self._write_failed = False if path is not None and path.exists(): self._repair_tail(path) self._catch_up() @@ -102,7 +126,7 @@ def _catch_up(self) -> None: data = handle.read() complete = data.rfind(b"\n") + 1 # All or nothing, or a retry after a bad line would count the good ones twice. - records = [json.loads(line) for line in data[:complete].splitlines() if line.strip()] + records = [_validated(json.loads(line)) for line in data[:complete].splitlines() if line.strip()] for record in records: self._add(record) self._read_up_to += complete @@ -117,8 +141,8 @@ def _add(self, record: dict) -> None: def _hourly_spend(self) -> float: cutoff = time.time() - 3600 - while self._recent and self._recent[0][0] < cutoff: - self._recent.popleft() + # Other processes and wall-clock changes can append records out of timestamp order. + self._recent = deque((stamp, cost) for stamp, cost in self._recent if stamp >= cutoff) return sum(cost for _, cost in self._recent) def session_spend(self, session_id: str) -> float: @@ -126,9 +150,12 @@ def session_spend(self, session_id: str) -> float: return self._session_spend.get(session_id, 0.0) def _check_locked(self, session_id: str, worst_case_usd: float) -> None: + _nonnegative(worst_case_usd) + if self._write_failed: + raise BudgetExceeded("ledger", self.project_spend, self.project_limit) try: self._catch_up() - except json.JSONDecodeError as exc: + except (ValueError, OSError) as exc: # We can't tell what has been spent, so nothing more gets spent. raise BudgetExceeded("ledger", self._project_spend, self.project_limit) from exc if worst_case_usd > self.request_limit: @@ -151,6 +178,13 @@ def check(self, session_id: str, worst_case_usd: float) -> None: with self._lock: self._check_locked(session_id, worst_case_usd) + def reserve_operation(self, session_id: str, worst_case_usd: float) -> AbstractContextManager[None]: + """File accounting reserves each attempt instead of prepaying retries.""" + return nullcontext() + + def reserve_attempt(self, session_id: str, worst_case_usd: float) -> AbstractContextManager[None]: + return self.reserve(session_id, worst_case_usd) + @contextmanager def reserve(self, session_id: str, worst_case_usd: float) -> Iterator[None]: """Hold a request's worst-case cost against the limits until it has finished.""" @@ -171,21 +205,23 @@ def reserve(self, session_id: str, worst_case_usd: float) -> Iterator[None]: del self._reserved_session[session_id] def record(self, record: UsageRecord) -> None: - payload = asdict(record) + payload = _validated(asdict(record)) with self._lock: if self.path is None: self._add(payload) return - self.path.parent.mkdir(parents=True, exist_ok=True) try: + self.path.parent.mkdir(parents=True, exist_ok=True) # First as well, so that a file replaced since the last look is noticed. self._catch_up() with self.path.open("a") as handle: handle.write(json.dumps(payload) + "\n") self._catch_up() - except json.JSONDecodeError: + except (ValueError, OSError): # The call has already happened, so count it, and the next check will refuse. self._add(payload) + self._write_failed = True + log.error("ledger_write_failed") def read_records(path: Path) -> list[UsageRecord]: diff --git a/services/api/quack/llm/postgres_ledger.py b/services/api/quack/llm/postgres_ledger.py index 2d69437..1b154ac 100644 --- a/services/api/quack/llm/postgres_ledger.py +++ b/services/api/quack/llm/postgres_ledger.py @@ -11,7 +11,7 @@ import time import uuid from collections.abc import Iterator -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager, nullcontext from contextvars import ContextVar from dataclasses import asdict from decimal import Decimal @@ -20,7 +20,7 @@ from psycopg.types.json import Jsonb from quack.db.database import Database -from quack.llm.ledger import BudgetExceeded, Ledger, UsageRecord +from quack.llm.ledger import BudgetExceeded, Ledger, UsageRecord, _nonnegative, _validated _active_debit: ContextVar[str | None] = ContextVar("quack_model_debit", default=None) _LOCK_ID = 743_812_095 @@ -36,8 +36,8 @@ def __init__( if not url: raise ValueError("PostgreSQL accounting requires QUACK_DATABASE_URL") for limit in (project_limit, session_limit, request_limit, hourly_limit): - if limit is not None and (not math.isfinite(limit) or limit < 0): - raise ValueError("model budgets must be finite and nonnegative") + if limit is not None: + _nonnegative(limit) self.path = None self.project_limit = project_limit self.session_limit = session_limit @@ -124,10 +124,23 @@ def _allocate(self, session_id: str, amount: float, *, debit: bool) -> str: def check(self, session_id: str, worst_case_usd: float) -> None: self._allocate(session_id, worst_case_usd, debit=False) + def reserve_operation(self, session_id: str, worst_case_usd: float) -> AbstractContextManager[None]: + """Keep the production operation's complete retry allowance durably prepaid.""" + return self.reserve(session_id, worst_case_usd) + + def reserve_attempt(self, session_id: str, worst_case_usd: float) -> AbstractContextManager[None]: + # Each attempt is already covered by the surrounding durable operation debit. + if _active_debit.get() is None or not self._healthy: + raise BudgetExceeded("ledger", 0, self.project_limit) + _nonnegative(worst_case_usd) + return nullcontext() + @contextmanager def reserve(self, session_id: str, worst_case_usd: float) -> Iterator[None]: from quack.llm.router import MAX_ATTEMPTS + _nonnegative(worst_case_usd) + # The existing router can retry an operation. Prepay its complete retry # allowance. Voice has no retry but uses the same conservative bound. # Never expire an unresolved debit. @@ -162,6 +175,11 @@ def _settle_known(self, debit_id: str) -> None: def record(self, record: UsageRecord) -> None: debit_id = _active_debit.get() + try: + _validated(asdict(record)) + except ValueError as exc: + self._healthy = False + raise BudgetExceeded("ledger", 0, self.project_limit) from exc if debit_id is None or not math.isfinite(record.estimated_usd) or record.estimated_usd < 0: raise BudgetExceeded("ledger", 0, self.project_limit) try: diff --git a/services/api/quack/llm/router.py b/services/api/quack/llm/router.py index 74d52ea..47542c3 100644 --- a/services/api/quack/llm/router.py +++ b/services/api/quack/llm/router.py @@ -8,14 +8,18 @@ import asyncio import json import logging +import math import random import time from dataclasses import dataclass, replace from enum import StrEnum -from typing import Any, Generic, TypeVar +from typing import Any, Generic, TypeVar, cast import openai from openai import AsyncOpenAI +from openai.types.chat import ChatCompletionMessageParam +from openai.types.shared import ReasoningEffort +from openai.types.shared_params import ResponseFormatJSONSchema from pydantic import BaseModel, ValidationError from quack.config import Settings @@ -49,6 +53,18 @@ class ModelRoute: # A student is waiting. After a timeout, the next pause tries again sooner than a retry would end. retry_timeouts: bool = True + def __post_init__(self) -> None: + if self.model not in ALLOWED_TOKEN_MODELS: + raise ValueError("model has no known price") + if self.reasoning_effort not in ("none", "minimal", "low", "medium", "high"): + raise ValueError("invalid reasoning effort") + if isinstance(self.timeout_s, bool) or not math.isfinite(self.timeout_s) or self.timeout_s <= 0: + raise ValueError("model timeout must be finite and positive") + if type(self.max_tokens) is not int or self.max_tokens <= 0: + raise ValueError("model token limit must be a positive integer") + if type(self.retry_timeouts) is not bool: + raise ValueError("retry_timeouts must be a boolean") + # Reading is timed per call in docs/IPAD_TEST_ROUND_2.md. The two readings run at different efforts # on purpose: identical ones could share a misreading, and the second is what stops an accusation. @@ -178,7 +194,7 @@ async def complete_json( client = self._get_client() worst_case = worst_case_output_cost(chosen.model, chosen.max_tokens) + ASSUMED_MAX_INPUT_USD try: - with self.ledger.reserve(session_id, worst_case): + with self.ledger.reserve_operation(session_id, worst_case): return await self._send(purpose, messages, schema, chosen, client, session_id, run_id) except BudgetExceeded as exc: log.warning("budget_exceeded", extra={"scope": exc.scope, "purpose": str(purpose)}) @@ -194,107 +210,117 @@ async def _send( session_id: str, run_id: str, ) -> StructuredResult[T]: - response_format = { + response_format: ResponseFormatJSONSchema = { "type": "json_schema", "json_schema": {"name": schema.__name__, "schema": schema.model_json_schema()}, } last_error: ModelError | None = None for attempt in range(1, MAX_ATTEMPTS + 1): - record = UsageRecord( - model=chosen.model, - reasoning_effort=chosen.reasoning_effort, - purpose=str(purpose), - run_id=run_id, - session_id=session_id, - attempt=attempt, - ) - started = time.perf_counter() - try: - response = await client.chat.completions.create( + # Each retry is another billable call. Re-check committed spend and reserve atomically. + worst_case = worst_case_output_cost(chosen.model, chosen.max_tokens) + ASSUMED_MAX_INPUT_USD + with self.ledger.reserve_attempt(session_id, worst_case): + record = UsageRecord( model=chosen.model, - messages=messages, # type: ignore[arg-type] - response_format=response_format, # type: ignore[arg-type] - max_tokens=chosen.max_tokens, - reasoning_effort=chosen.reasoning_effort, # type: ignore[arg-type] - timeout=chosen.timeout_s, - extra_body={"prompt_cache_key": f"quack-{schema.__name__}"}, + reasoning_effort=chosen.reasoning_effort, + purpose=str(purpose), + run_id=run_id, + session_id=session_id, + attempt=attempt, ) - except (openai.APIError, TimeoutError) as exc: - category, retryable, status = _categorise(exc) - record.elapsed_ms = (time.perf_counter() - started) * 1000 - record.http_status = status - record.error_category = category - # The model may have done the work and billed for it, with the reply lost on the way. - # A refused connection never reached it, so that costs nothing. - if isinstance(exc, openai.APITimeoutError | TimeoutError): - record.estimated_usd = worst_case_output_cost(chosen.model, chosen.max_tokens) - self.ledger.record(record) - log.warning("model_request_failed", extra={"category": category, "status": status, "attempt": attempt}) - last_error = ModelError(category, f"model request failed: {category}", status) - if category == "timeout" and not chosen.retry_timeouts: + started = time.perf_counter() + try: + response = await client.chat.completions.create( + model=chosen.model, + messages=cast(list[ChatCompletionMessageParam], messages), + response_format=response_format, + max_tokens=chosen.max_tokens, + reasoning_effort=cast(ReasoningEffort, chosen.reasoning_effort), + timeout=chosen.timeout_s, + extra_body={"prompt_cache_key": f"quack-{schema.__name__}"}, + ) + except asyncio.CancelledError: + # Cancelling our wait doesn't prove the provider stopped its billable work. + record.elapsed_ms = (time.perf_counter() - started) * 1000 + record.error_category = "cancelled" + record.estimated_usd = worst_case + self.ledger.record(record) + log.info("model_request_cancelled", extra={"purpose": str(purpose), "attempt": attempt}) + raise + except (openai.APIError, TimeoutError) as exc: + category, retryable, status = _categorise(exc) + record.elapsed_ms = (time.perf_counter() - started) * 1000 + record.http_status = status + record.error_category = category + # The model may have done the work and billed for it, with the reply lost on the way. + # A refused connection never reached it, so that costs nothing. + if isinstance(exc, openai.APITimeoutError | TimeoutError): + record.estimated_usd = worst_case_output_cost(chosen.model, chosen.max_tokens) + self.ledger.record(record) + log.warning("model_request_failed", extra={"category": category, "status": status, "attempt": attempt}) + last_error = ModelError(category, f"model request failed: {category}", status) + if category == "timeout" and not chosen.retry_timeouts: + raise last_error from exc + if retryable and attempt < MAX_ATTEMPTS: + delay = _retry_delay(attempt, exc) + if delay is not None: + await asyncio.sleep(delay) + continue raise last_error from exc - if retryable and attempt < MAX_ATTEMPTS: - delay = _retry_delay(attempt, exc) - if delay is not None: - await asyncio.sleep(delay) - continue - raise last_error from exc - - record.elapsed_ms = (time.perf_counter() - started) * 1000 - record.http_status = 200 - for key, value in _usage_fields(response.usage).items(): - setattr(record, key, value) - record.estimated_usd = ( - token_cost(chosen.model, record.input_tokens, record.cached_input_tokens, record.output_tokens) - if response.usage is not None - # It says nothing about what it cost. A budget mustn't take that to mean nothing. - else worst_case_output_cost(chosen.model, chosen.max_tokens) - ) - if not response.choices: - record.error_category = "invalid_output" - self.ledger.record(record) - log.warning("model_output_invalid", extra={"attempt": attempt, "purpose": str(purpose)}) - last_error = ModelError("invalid_output", "model response had no choices") - if attempt < MAX_ATTEMPTS - 1: - continue - raise last_error - - choice = response.choices[0] - if choice.finish_reason == "length": - # Reasoning shares the token limit, so a retry at the same limit tends to fail - # the same way and costs the same again. - record.error_category = "truncated" - self.ledger.record(record) - raise ModelError("truncated", "output hit max_tokens before finishing") + record.elapsed_ms = (time.perf_counter() - started) * 1000 + record.http_status = 200 + for key, token_count in _usage_fields(response.usage).items(): + setattr(record, key, token_count) + record.estimated_usd = ( + token_cost(chosen.model, record.input_tokens, record.cached_input_tokens, record.output_tokens) + if response.usage is not None + # It says nothing about what it cost. A budget mustn't take that to mean nothing. + else worst_case_output_cost(chosen.model, chosen.max_tokens) + ) + + if not response.choices: + record.error_category = "invalid_output" + self.ledger.record(record) + log.warning("model_output_invalid", extra={"attempt": attempt, "purpose": str(purpose)}) + last_error = ModelError("invalid_output", "model response had no choices") + if attempt < MAX_ATTEMPTS - 1: + continue + raise last_error + + choice = response.choices[0] + if choice.finish_reason == "length": + # Reasoning shares the token limit, so a retry at the same limit tends to fail + # the same way and costs the same again. + record.error_category = "truncated" + self.ledger.record(record) + raise ModelError("truncated", "output hit max_tokens before finishing") + + try: + value = schema.model_validate_json(choice.message.content or "") + except ValidationError as exc: + record.error_category = "invalid_output" + self.ledger.record(record) + log.warning("model_output_invalid", extra={"attempt": attempt, "purpose": str(purpose)}) + last_error = ModelError("invalid_output", "model output failed schema validation") + if attempt < MAX_ATTEMPTS - 1: + continue + raise last_error from exc - try: - value = schema.model_validate_json(choice.message.content or "") - except ValidationError as exc: - record.error_category = "invalid_output" self.ledger.record(record) - log.warning("model_output_invalid", extra={"attempt": attempt, "purpose": str(purpose)}) - last_error = ModelError("invalid_output", "model output failed schema validation") - if attempt < MAX_ATTEMPTS - 1: - continue - raise last_error from exc - - self.ledger.record(record) - log.info( - "model_request", - extra={ - "purpose": str(purpose), - "model": chosen.model, - "effort": chosen.reasoning_effort, - "ms": round(record.elapsed_ms), - "input": record.input_tokens, - "cached": record.cached_input_tokens, - "output": record.output_tokens, - "usd": round(record.estimated_usd, 6), - }, - ) - return StructuredResult(value, record) + log.info( + "model_request", + extra={ + "purpose": str(purpose), + "model": chosen.model, + "effort": chosen.reasoning_effort, + "ms": round(record.elapsed_ms), + "input": record.input_tokens, + "cached": record.cached_input_tokens, + "output": record.output_tokens, + "usd": round(record.estimated_usd, 6), + }, + ) + return StructuredResult(value, record) - assert last_error is not None - raise last_error + raise last_error or ModelError("invalid_output", "model request produced no result") diff --git a/services/api/quack/main.py b/services/api/quack/main.py index c901388..8c07a10 100644 --- a/services/api/quack/main.py +++ b/services/api/quack/main.py @@ -4,7 +4,6 @@ import asyncio import ipaddress -import json import logging import sys import time @@ -28,6 +27,7 @@ from quack.llm.postgres_ledger import PostgresLedger from quack.llm.report import summarise from quack.llm.router import ModelError, ModelRouter +from quack.observability import JsonFormatter, RequestDiagnostics from quack.schemas import ( AnalyzeRequest, AnalyzeResponse, @@ -63,7 +63,6 @@ from quack.services.voice import VoiceRejected, classify_intent, transcribe_wav from quack.services.writer_references import select_references -_STANDARD_LOG_FIELDS = set(logging.makeLogRecord({}).__dict__) | {"message", "asctime", "taskName"} MAX_REQUEST_BYTES = 10_000_000 MAX_AUDIO_BYTES = 8_000_000 MAX_TRACKED_CLIENTS = 10_000 @@ -73,13 +72,6 @@ EXPLAIN_DEADLINE_S = 25.0 -class JsonFormatter(logging.Formatter): - def format(self, record: logging.LogRecord) -> str: - payload = {"level": record.levelname, "logger": record.name, "event": record.getMessage()} - payload.update({k: v for k, v in record.__dict__.items() if k not in _STANDARD_LOG_FIELDS}) - return json.dumps(payload, default=str) - - def _configure_logging() -> None: handler = logging.StreamHandler(sys.stdout) handler.setFormatter(JsonFormatter()) @@ -148,7 +140,12 @@ def _same_origin(request: Request, allowed: list[str]) -> bool: return True # Host can be anything under DNS rebinding, where a site's name is pointed at this machine. So # beyond the configured origins, a match only counts on a local or private address. - parts = urlsplit(origin) + try: + parts = urlsplit(origin) + except ValueError: + return False + if parts.scheme not in ("http", "https") or parts.username or parts.password: + return False hosts = (request.headers.get("host"), request.headers.get("x-forwarded-host")) return _is_local_host(parts.hostname) and parts.netloc in hosts @@ -254,7 +251,7 @@ def _assistance( level: int, text: str, source: Literal["model", "template"], error: ModelError | None, started: float ) -> AssistanceResponse: total_ms = round((time.perf_counter() - started) * 1000, 1) - logging.getLogger("quack.help").info("help", extra={"level": level, "source": source, "total_ms": total_ms}) + logging.getLogger("quack.help").info("help", extra={"help_level": level, "source": source, "total_ms": total_ms}) return AssistanceResponse( level=level, text=text, source=source, model_issue=model_issue(error) if error else None, total_ms=total_ms ) @@ -285,6 +282,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: allow_origins=state.settings.cors_origin_list, allow_methods=["GET", "POST", "PUT", "DELETE"], allow_headers=["content-type", "authorization"], + expose_headers=["x-request-id"], ) app.add_middleware(BodySizeLimit, limit=MAX_REQUEST_BYTES) @@ -301,6 +299,8 @@ async def reject_cross_site(request: Request, call_next): # type: ignore[no-unt return JSONResponse({"detail": "cross-site request refused"}, status_code=403) return await call_next(request) + app.add_middleware(RequestDiagnostics) + def rate_limited(request: Request) -> None: client = request.client.host if request.client else "unknown" if not state.limiter.allow(client): diff --git a/services/api/quack/observability.py b/services/api/quack/observability.py new file mode 100644 index 0000000..b99c7a9 --- /dev/null +++ b/services/api/quack/observability.py @@ -0,0 +1,88 @@ +"""Request diagnostics with no bodies, credentials, user ids, or literal URL paths.""" + +from __future__ import annotations + +import json +import logging +import time +import traceback +from contextvars import ContextVar +from datetime import UTC, datetime +from pathlib import Path +from uuid import uuid4 + +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +request_id: ContextVar[str | None] = ContextVar("request_id", default=None) +log = logging.getLogger("quack.http") +_STANDARD_FIELDS = set(logging.makeLogRecord({}).__dict__) | {"message", "asctime", "taskName"} + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload = { + "timestamp": datetime.fromtimestamp(record.created, UTC).isoformat(), + "level": record.levelname, + "logger": record.name, + "event": record.getMessage(), + **{k: v for k, v in record.__dict__.items() if k not in _STANDARD_FIELDS}, + } + if current := request_id.get(): + payload["request_id"] = current + return json.dumps(payload, default=str, allow_nan=False) + + +class RequestDiagnostics: + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + # Always server-generated: a caller cannot inject log content or reuse another request's id. + correlation = uuid4().hex + token = request_id.set(correlation) + started = time.perf_counter() + status = 500 + response_started = False + + async def observed(message: Message) -> None: + nonlocal status, response_started + if message["type"] == "http.response.start": + status, response_started = message["status"], True + headers = list(message.get("headers", [])) + headers = [(k, v) for k, v in headers if k.lower() != b"x-request-id"] + headers.append((b"x-request-id", correlation.encode("ascii"))) + if scope.get("path", "").startswith("/api/"): + headers = [(k, v) for k, v in headers if k.lower() != b"cache-control"] + headers.append((b"cache-control", b"no-store")) + message = {**message, "headers": headers} + await send(message) + + try: + try: + await self.app(scope, receive, observed) + except Exception as exc: + # Exception messages can contain model text, SQL parameters or credentials. + frames = [ + {"file": Path(frame.f_code.co_filename).name, "line": line, "function": frame.f_code.co_name} + for frame, line in traceback.walk_tb(exc.__traceback__) + ][-8:] + log.error("request_failed", extra={"error_type": type(exc).__name__, "frames": frames}) + if response_started: + raise + await JSONResponse({"detail": "internal server error"}, status_code=500)(scope, receive, observed) + finally: + route = scope.get("route") + log.info( + "http_request", + extra={ + "method": scope["method"], + "route": getattr(route, "path", "unmatched"), + "status": status, + "duration_ms": round((time.perf_counter() - started) * 1000, 2), + }, + ) + request_id.reset(token) diff --git a/services/api/quack/schemas.py b/services/api/quack/schemas.py index df769af..0b659c2 100644 --- a/services/api/quack/schemas.py +++ b/services/api/quack/schemas.py @@ -2,9 +2,9 @@ from __future__ import annotations -from typing import Annotated, Any, Literal +from typing import Annotated, Any, Literal, Self -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from pydantic.alias_generators import to_camel AnalysisStatus = Literal["no_error", "confirmed_error", "uncertain", "unsupported"] @@ -210,6 +210,27 @@ class AnalyzeResponse(ApiModel): # Right as far as it goes, and with something asked for still to add: "missing_constant". reminder: str | None = None + @model_validator(mode="after") + def enforce_alert_evidence(self) -> Self: + """Refuse internally inconsistent alerts before they can reach a student's notebook. + + These checks run under python -O too; release safety cannot depend on assert statements. + The verifier remains the only source of the `verified` evidence itself. + """ + + ids = {mistake.id for mistake in self.mistakes} + if len(ids) != len(self.mistakes) or not all(mistake.verified for mistake in self.mistakes): + raise ValueError("mistakes must have unique ids and verified evidence") + if self.mistake is not None and (self.mistake not in self.mistakes or not self.mistake.verified): + raise ValueError("selected mistake must belong to the verified mistakes") + if ids.intersection(self.resolved_mistake_ids): + raise ValueError("a mistake cannot be present and resolved in the same response") + if self.intervention.should_quack and ( + self.status != "confirmed_error" or self.stale or not any(mistake.is_new for mistake in self.mistakes) + ): + raise ValueError("an alert needs a current, new, verified mistake") + return self + class ClearRequest(ApiModel): session_id: str = Field(min_length=1, max_length=64) diff --git a/services/api/quack/services/analysis.py b/services/api/quack/services/analysis.py index 5500a44..af166f4 100644 --- a/services/api/quack/services/analysis.py +++ b/services/api/quack/services/analysis.py @@ -1164,7 +1164,7 @@ def _lay_out( remembered = self._memory.get(session_id, ink.id) line = just_read if reread and just_read is not None else remembered or just_read if ( - overruled is not None and line is remembered and just_read is not None + overruled is not None and remembered is not None and line is remembered and just_read is not None and remembered.kind == just_read.kind == "math" and not _same_answer(remembered.math, just_read.math) ): overruled.add(ink.id) diff --git a/services/api/quack/services/handwriting.py b/services/api/quack/services/handwriting.py index ffe841e..a2e0538 100644 --- a/services/api/quack/services/handwriting.py +++ b/services/api/quack/services/handwriting.py @@ -6,7 +6,7 @@ import hashlib import io from dataclasses import dataclass -from typing import Literal +from typing import Any, Literal from PIL import Image, UnidentifiedImageError from pydantic import BaseModel, ConfigDict @@ -113,8 +113,9 @@ def sanitise_image(data_url: str) -> tuple[str, str]: if width * height > MAX_IMAGE_PIXELS: raise ImageRejected("image has too many pixels") probe.verify() - with Image.open(io.BytesIO(raw)) as image: - image.load() + with Image.open(io.BytesIO(raw)) as source: + source.load() + image: Image.Image = source if image.mode in ("RGBA", "LA", "P"): rgba = image.convert("RGBA") flattened = Image.new("RGB", rgba.size, "white") @@ -161,7 +162,7 @@ async def read_lines( content.append({"type": "text", "text": f"{lead} It has {line_count} written {noun}."}) content.append({"type": "image_url", "image_url": {"url": image}}) instructions = f"{COMPACT_PROMPT}\n\n{WRITER_REFERENCES_PROMPT}" if references else COMPACT_PROMPT - messages = [ + messages: list[dict[str, Any]] = [ {"role": "developer", "content": instructions}, {"role": "user", "content": content}, ] @@ -195,7 +196,7 @@ async def read_handwriting( } ) content.append({"type": "image_url", "image_url": {"url": page_context}}) - messages = [ + messages: list[dict[str, Any]] = [ # Stable instructions first, so that automatic prefix caching applies. {"role": "developer", "content": DEVELOPER_PROMPT}, {"role": "user", "content": content}, diff --git a/services/api/quack/services/hint_engine.py b/services/api/quack/services/hint_engine.py index 70d7174..33dcf05 100644 --- a/services/api/quack/services/hint_engine.py +++ b/services/api/quack/services/hint_engine.py @@ -972,4 +972,4 @@ async def get(self, session_id: str, mistake: TrackedMistake, level: Literal[1, def _log_failure(task: asyncio.Task[HelpResult]) -> None: if not task.cancelled() and task.exception() is not None: - log.warning("hint_preparation_failed", extra={"error": repr(task.exception())}) + log.warning("hint_preparation_failed", extra={"error_type": type(task.exception()).__name__}) diff --git a/services/api/quack/services/math_verifier.py b/services/api/quack/services/math_verifier.py index 3a12d5c..c44cbea 100644 --- a/services/api/quack/services/math_verifier.py +++ b/services/api/quack/services/math_verifier.py @@ -1226,7 +1226,7 @@ def _given(values: Values) -> dict[sp.Symbol, sp.Expr]: def _mentions_a_given(left: Node, right: Node, values: Values) -> bool: - if any(contains_kind(node, kind) for node in (left, right) for kind in ("deriv", "integral")): + if any(contains_kind(node, "deriv") or contains_kind(node, "integral") for node in (left, right)): return False letters = {letter for letter, _ in values} return bool(letters & (_written_variables(left) | _written_variables(right))) @@ -1826,9 +1826,10 @@ def _check_equation( if cur_values is not None: after_roots = cur_values else: - after_roots = _real_solutions(lhs_cur, rhs_cur, variable) - if after_roots is None: + solved_roots = _real_solutions(lhs_cur, rhs_cur, variable) + if solved_roots is None: return StepResult("unsupported", operation, line, from_line, reason="not_a_rational_equation") + after_roots = solved_roots same, after_within_before, before_within_after = _same_set(after_roots, before_roots) if same: @@ -1892,7 +1893,7 @@ def _check_equation( fingerprint=_fingerprint( "equation_solving", _canonical_equation(before), - _canonical_equation(after) if after is not None else sp.Tuple(*cur_values), + _canonical_equation(after) if after is not None else sp.Tuple(*(cur_values or [])), ), ) @@ -2128,14 +2129,15 @@ def _looks_like_identity_claim(left: Node, right: Node, line: LineInput, prior: except MathParseError: return False # `1 + 1 = 2` poses nothing to solve, so what comes after it isn't a step of solving it. - prior_equation = _is_equation(prior) and not _is_numeric_statement(prior) + prior_equation = prior is not None and _is_equation(prior) and not _is_numeric_statement(prior) if not a.free_symbols and not b.free_symbols: # After an equation, `1 = 2` is a result (no solutions) rather than a sum to check. if not prior_equation: return True - if _has_nothing_to_work_out(prior) and left.kind in ("add", "mul", "div", "pow") and _is_literal(right): + if (prior is not None and _has_nothing_to_work_out(prior) + and left.kind in ("add", "mul", "div", "pow") and _is_literal(right)): return True - return line.continues == "no" and not _could_follow_without_being_derived(prior, left, right) + return prior is not None and line.continues == "no" and not _could_follow_without_being_derived(prior, left, right) if (prior_equation and line.continues != "no") or line.operation not in SIMPLIFY_OPERATIONS: return False if sp.expand(a).free_symbols != b.free_symbols: @@ -2591,15 +2593,15 @@ def clears(context: _Context, stmt: Statement) -> bool: for prev in previous_readings: anchor = prev if ctx.anchor is ctx.prev else ctx.anchor context = replace(ctx, prev=prev, anchor=anchor) - for stmt in [original, *current_readings]: - if stmt is not None and clears(context, stmt): + for alternative in [original, *current_readings]: + if alternative is not None and clears(context, alternative): return True # An equation step can be compared with a line further back than the one above. if ctx.anchor_input is not None and ctx.anchor_input is not ctx.prev_line: for anchor in _parse_alternatives(ctx.anchor_input): context = replace(ctx, anchor=anchor) - for stmt in [original, *current_readings]: - if stmt is not None and clears(context, stmt): + for alternative in [original, *current_readings]: + if alternative is not None and clears(context, alternative): return True return False diff --git a/services/api/quack/services/practice.py b/services/api/quack/services/practice.py index a74059c..58d878e 100644 --- a/services/api/quack/services/practice.py +++ b/services/api/quack/services/practice.py @@ -14,13 +14,14 @@ from dataclasses import dataclass from fractions import Fraction +from quack.schemas import PracticeTopic from quack.services.handwriting import RESTATED_QUESTION, HandwritingReading, ReadLine from quack.services.math_parser import MathParseError, normalise, parse_expression, parse_statement from quack.services.math_verifier import LineInput, TaskIntent, names_given_values from quack.services.verifier_pool import VerificationFailed, shared_pool MAX_TRIES = 8 -SUPPORTED = ( +SUPPORTED: tuple[PracticeTopic, ...] = ( "addition_subtraction", "multiplication", "division", diff --git a/services/api/quack/services/starter_questions.py b/services/api/quack/services/starter_questions.py index 2582bc3..dbdba6a 100644 --- a/services/api/quack/services/starter_questions.py +++ b/services/api/quack/services/starter_questions.py @@ -17,6 +17,7 @@ import random from fractions import Fraction +from functools import partial from typing import Any from quack.services.lesson_questions import ( @@ -29,6 +30,7 @@ TOPIC_OF_SKILL, LessonQuestion, Template, + Tier, _lesson, _pick, ) @@ -245,7 +247,7 @@ def _proper(rng: random.Random, fixed: dict[str, Any] | None, largest: int) -> t return top, bottom -def _simplify(template: str, tier: str, factors: tuple[int, int]): # type: ignore[no-untyped-def] +def _simplify(template: str, tier: Tier, factors: tuple[int, int]): # type: ignore[no-untyped-def] def build(rng: random.Random, fixed: dict[str, Any] | None) -> LessonQuestion: top, bottom = (fixed["p"], fixed["q"]) if fixed and "p" in fixed else _proper(rng, fixed, 7) k = _pick(rng, fixed, "k", lambda: rng.randint(*factors)) @@ -340,7 +342,7 @@ def _subtract_fractions(rng: random.Random, fixed: dict[str, Any] | None) -> Les def _to_decimal(rng: random.Random, fixed: dict[str, Any] | None) -> LessonQuestion: while True: bottom = _pick(rng, fixed, "q", lambda: rng.choice(_DECIMAL_BOTTOMS)) - top = _pick(rng, fixed, "p", lambda bottom=bottom: rng.randint(1, bottom - 1)) + top = _pick(rng, fixed, "p", partial(rng.randint, 1, bottom - 1)) if Fraction(top, bottom).denominator == bottom: break fixed = None @@ -361,7 +363,7 @@ def _to_decimal(rng: random.Random, fixed: dict[str, Any] | None) -> LessonQuest def _to_fraction(rng: random.Random, fixed: dict[str, Any] | None) -> LessonQuestion: while True: bottom = _pick(rng, fixed, "q", lambda: rng.choice((4, 5, 20, 25))) - top = _pick(rng, fixed, "p", lambda bottom=bottom: rng.randint(1, bottom - 1)) + top = _pick(rng, fixed, "p", partial(rng.randint, 1, bottom - 1)) if Fraction(top, bottom).denominator == bottom: break fixed = None @@ -518,7 +520,7 @@ def _odd_triangle_area(rng: random.Random, fixed: dict[str, Any] | None) -> Less _RATIO_SIDES = {"sin": ("opposite", "hypotenuse"), "cos": ("adjacent", "hypotenuse"), "tan": ("opposite", "adjacent")} -def _ratio(template: str, tier: str, ratios: tuple[str, ...]): # type: ignore[no-untyped-def] +def _ratio(template: str, tier: Tier, ratios: tuple[str, ...]): # type: ignore[no-untyped-def] def build(rng: random.Random, fixed: dict[str, Any] | None) -> LessonQuestion: opposite, adjacent, hypotenuse = _pick(rng, fixed, "triple", lambda: rng.choice(_TRIPLES)) ratio = _pick(rng, fixed, "ratio", lambda: rng.choice(ratios)) diff --git a/services/api/quack/services/verifier_pool.py b/services/api/quack/services/verifier_pool.py index df5bfe1..f584d27 100644 --- a/services/api/quack/services/verifier_pool.py +++ b/services/api/quack/services/verifier_pool.py @@ -69,7 +69,8 @@ def _child(conn: Connection, job: object, *args: object) -> None: try: conn.send(("ok", job(*args))) # type: ignore[operator] except BaseException as exc: # noqa: BLE001 - conn.send(("error", repr(exc))) + # Exception messages can include the student's parsed expression. + conn.send(("error", type(exc).__name__)) finally: conn.close() @@ -118,7 +119,7 @@ def _run_job(self, job: object, args: tuple[object, ...], timeout_s: float): # process.join(1) if status != "ok": # A bug in checking one line shouldn't take down analysis. Quack just stays quiet. - log.error("verification_error", extra={"error": payload}) + log.error("verification_error", extra={"error_type": payload}) raise VerificationFailed("error") return payload diff --git a/services/api/requirements-dev.txt b/services/api/requirements-dev.txt index bedd186..96b2efa 100644 --- a/services/api/requirements-dev.txt +++ b/services/api/requirements-dev.txt @@ -4,3 +4,4 @@ pytest-asyncio>=0.24,<2 respx>=0.21,<1 ruff>=0.6,<1 mypy>=1.11,<2 +-r requirements-deploy.txt diff --git a/services/api/requirements-tooling.lock b/services/api/requirements-tooling.lock new file mode 100644 index 0000000..0269773 --- /dev/null +++ b/services/api/requirements-tooling.lock @@ -0,0 +1,20 @@ +# Pinned deployment-test tooling. Install with requirements.lock. +aiohappyeyeballs==2.7.1 +aiohttp==3.14.5 +aiosignal==1.4.0 +attrs==26.1.0 +cbor2==6.2.1 +frozenlist==1.8.0 +grpclib==0.4.9 +h2==4.4.1 +hpack==4.2.0 +hyperframe==6.1.0 +modal==1.6.3 +multidict==7.2.1 +propcache==0.5.4 +protobuf==7.36.2 +synchronicity==0.12.7 +toml==0.10.2 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260518 +yarl==1.25.1 diff --git a/services/api/tests/accounts_support.py b/services/api/tests/accounts_support.py index 141e72e..0291070 100644 --- a/services/api/tests/accounts_support.py +++ b/services/api/tests/accounts_support.py @@ -40,6 +40,9 @@ def _empty_postgres() -> None: return import psycopg + info = psycopg.conninfo.conninfo_to_dict(postgres) + if info.get("dbname") != "quack_accounts_test" or info.get("host") not in {"127.0.0.1", "localhost", "::1"}: + pytest.fail("account tests require a local disposable database named quack_accounts_test") with psycopg.connect(postgres, autocommit=True) as connection: # Each test's server leaves a connection open, and they'd soon use up the limit. connection.execute("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid()") diff --git a/services/api/tests/conftest.py b/services/api/tests/conftest.py index 2b15a20..2e327cf 100644 --- a/services/api/tests/conftest.py +++ b/services/api/tests/conftest.py @@ -1,3 +1,4 @@ +import os from pathlib import Path import pytest @@ -6,6 +7,23 @@ from tests.accounts_support import _empty_postgres, accounts_client # noqa: F401 +def pytest_sessionfinish(session: pytest.Session) -> None: + if os.environ.get("QUACK_RELEASE_VERIFY") == "1": + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter and reporter.stats.get("skipped"): + reporter.write_sep("=", "Release verification refuses skipped tests") + session.exitstatus = pytest.ExitCode.TESTS_FAILED + + +@pytest.fixture(autouse=True) +def isolate_mocked_http(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: + """Mocked HTTP must not depend on a developer's or CI runner's proxy setup.""" + + if request.node.get_closest_marker("live") is None: + for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + monkeypatch.delenv(name, raising=False) + + @pytest.fixture def settings(tmp_path: Path) -> Settings: return Settings( diff --git a/services/api/tests/fixtures/recorded-routing.json b/services/api/tests/fixtures/recorded-routing.json new file mode 100644 index 0000000..dc94418 --- /dev/null +++ b/services/api/tests/fixtures/recorded-routing.json @@ -0,0 +1,371 @@ +{ + "description": "Historical classifications on the published synthetic stroke dataset. Only case/config/status and hashed fingerprint equality are retained; no model text, usage, paths or student records. These are historical prompt regressions, not a current accuracy benchmark.", + "runs": { + "v5-all": [ + {"case":"st_v_mul_3x4","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_mul_2x3","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_v_mul_2x3","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_mul_3x4","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_i_bigx_2x3","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_i_bigx_3x4","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_v_bigx_3x4","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_bigx_2x3","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_add","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"eb06db0df854904167205a7daf662e7fd4aa3320012f0c59031342c71773a770"}, + {"case":"st_v_add","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_div","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_div","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c552db87e0f43b07e518c459317f3eb5306b5890b1f27f464d0cf184650765d1"}, + {"case":"st_i_diff_power","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"27fc1e0276480da656072b5157049c00d2a5279a4d8c9a3f3a594d18e4ed8c80"}, + {"case":"st_i_dist","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"st_i_diff","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"st_v_diff","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"demo_algebra","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_correct","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_expand","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_two_brackets","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"demo_diff","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"alg_v_negative","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_like_terms","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_square","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_factor","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_arith","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_chain","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_unconventional","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_equation","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_distribution","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e1141c11f1d72004b667721d625457f4bedf70061aee83121e30ab073bcee9bf"}, + {"case":"alg_i_square","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"5a2ebe1cd6298981aace9f4d01b84eb55ef82f379dbf737829f03befe791efd1"}, + {"case":"alg_v_quadratic","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_arith","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"58442325e681a7eb66ad433444459dd70d751a40d240eba3af1eb90321dcc642"}, + {"case":"alg_i_like_terms","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"0981a9217075da65f72d8244a5c881469ede06e30446538f37a2b630e83a05d8"}, + {"case":"alg_i_sign","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"b3b93fb297a94023701b9c9ec6bc8655b354e5013c3699eab331ce8b135e00c3"}, + {"case":"alg_i_two_brackets","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"3a3fac44294f97ac1ee410a45acf9b102d238561f408ee18ddb439bdfad10bfa"}, + {"case":"alg_i_equation_divide","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"824ce6da1f0844ac7f323307b23909dc3c26d1eb901128347bd6aaa76e27be72"}, + {"case":"alg_i_factor","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c735cb26eea690d7a3c52fae89fc558c3faa4ed7fdc10cb4e5c93360c3d8fd89"}, + {"case":"alg_i_equation_sign","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"18d193684972a86c8ddd857fff19d2182e3b3a3fddcb06472f44bca51e3d56b8"}, + {"case":"alg_i_later_step","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"90e3f414eb263119efa29fffd90be12fe63b281b4c9935e8ed0eda4abaecb114"}, + {"case":"diff_v_constant","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_power","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sum","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_poly","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_leibniz","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_function","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sin","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_i_exponent","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e6ffbc72abc45894e8b6f38505d7f90486f297d9ba39717be4e422acd8929116"}, + {"case":"diff_v_chain","config":"A","expected":"no_error","predicted":"uncertain","fingerprint":null}, + {"case":"diff_i_constant","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"4dcfc8cec5ae4ccc1f6faec2bb3352dd946d8e6f76d1847bf7bdd201e4881f9e"}, + {"case":"diff_i_coefficient","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"7ef1a56bbbe0fbe7adf87bd8ef561f8e7d06dda562b5a31977178227309a8e4f"}, + {"case":"diff_i_missing","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"befe77023a414fae34dadba3da42915142362779cd01d5d274102d7a8fc1e833"}, + {"case":"diff_i_function","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"24cc3dea012470a6f45e7539700f9a0d8cea6e0d2ab94b25c38aec3e984c9320"}, + {"case":"diff_i_leibniz","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a095ec1e895734cdb3ab5ec498bc4a7ab1f69449462c55ce92b77f9994cacfeb"}, + {"case":"diff_i_sin","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"6910e10d635fa5a411494361dd0eef03073af077a0c7f1ead135569ec10d433b"}, + {"case":"diff_i_chain","config":"A","expected":"confirmed_error","predicted":"uncertain","fingerprint":null}, + {"case":"abs_incomplete","config":"A","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_integral","config":"A","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_inequality","config":"A","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_substitution","config":"A","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_lost_root","config":"A","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_messy_invalid","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"ac8bfef7ed8c7604698f84532321cc417367a6f39c5d9c449df1aa8b1fc37d30"}, + {"case":"abs_domain_sqrt","config":"A","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_overlap_invalid","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"47d7aa89f19bf83ad6b1a1dc70dc553c1379fcb0c82797948cdb344d0ba7c61f"}, + {"case":"hard_overlap_valid","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_messy_valid","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_valid","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_fraction_valid","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_fraction_invalid","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"d7c9aab23af77e60f44face279dacc6d8fb39fe4e1307314c1ae3563e67b16bf"}, + {"case":"hard_super_invalid","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"1e1488af4943af3c3ad28c02fe8ee23b5e741eef72bc1355f31135df6499f994"}, + {"case":"hard_crossed_out","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_two_errors","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"} + ], + "v6-compact": [ + {"case":"st_i_mul_2x3","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_v_mul_2x3","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_mul_3x4","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_mul_3x4","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_i_bigx_2x3","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_v_bigx_2x3","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_bigx_3x4","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_div","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c552db87e0f43b07e518c459317f3eb5306b5890b1f27f464d0cf184650765d1"}, + {"case":"st_i_bigx_3x4","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_v_div","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_add","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"eb06db0df854904167205a7daf662e7fd4aa3320012f0c59031342c71773a770"}, + {"case":"st_v_add","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_diff","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"st_v_diff","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_diff_power","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"27fc1e0276480da656072b5157049c00d2a5279a4d8c9a3f3a594d18e4ed8c80"}, + {"case":"st_i_dist","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_diff","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"demo_algebra","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_correct","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_expand","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_two_brackets","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_negative","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_like_terms","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_square","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_factor","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_chain","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_arith","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_equation","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_unconventional","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_distribution","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e1141c11f1d72004b667721d625457f4bedf70061aee83121e30ab073bcee9bf"}, + {"case":"alg_v_quadratic","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_square","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"5a2ebe1cd6298981aace9f4d01b84eb55ef82f379dbf737829f03befe791efd1"}, + {"case":"alg_i_arith","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"58442325e681a7eb66ad433444459dd70d751a40d240eba3af1eb90321dcc642"}, + {"case":"alg_i_sign","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"b3b93fb297a94023701b9c9ec6bc8655b354e5013c3699eab331ce8b135e00c3"}, + {"case":"alg_i_two_brackets","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"3a3fac44294f97ac1ee410a45acf9b102d238561f408ee18ddb439bdfad10bfa"}, + {"case":"alg_i_like_terms","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"0981a9217075da65f72d8244a5c881469ede06e30446538f37a2b630e83a05d8"}, + {"case":"alg_i_factor","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c735cb26eea690d7a3c52fae89fc558c3faa4ed7fdc10cb4e5c93360c3d8fd89"}, + {"case":"alg_i_equation_sign","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"18d193684972a86c8ddd857fff19d2182e3b3a3fddcb06472f44bca51e3d56b8"}, + {"case":"alg_i_equation_divide","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"824ce6da1f0844ac7f323307b23909dc3c26d1eb901128347bd6aaa76e27be72"}, + {"case":"alg_i_later_step","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"90e3f414eb263119efa29fffd90be12fe63b281b4c9935e8ed0eda4abaecb114"}, + {"case":"diff_v_sum","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_power","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_constant","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_poly","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_leibniz","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_function","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sin","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_i_exponent","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e6ffbc72abc45894e8b6f38505d7f90486f297d9ba39717be4e422acd8929116"}, + {"case":"diff_v_chain","config":"H","expected":"no_error","predicted":"uncertain","fingerprint":null}, + {"case":"diff_i_coefficient","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"7ef1a56bbbe0fbe7adf87bd8ef561f8e7d06dda562b5a31977178227309a8e4f"}, + {"case":"diff_i_constant","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"4dcfc8cec5ae4ccc1f6faec2bb3352dd946d8e6f76d1847bf7bdd201e4881f9e"}, + {"case":"diff_i_missing","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"befe77023a414fae34dadba3da42915142362779cd01d5d274102d7a8fc1e833"}, + {"case":"diff_i_leibniz","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a095ec1e895734cdb3ab5ec498bc4a7ab1f69449462c55ce92b77f9994cacfeb"}, + {"case":"diff_i_function","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"24cc3dea012470a6f45e7539700f9a0d8cea6e0d2ab94b25c38aec3e984c9320"}, + {"case":"diff_i_sin","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"6910e10d635fa5a411494361dd0eef03073af077a0c7f1ead135569ec10d433b"}, + {"case":"abs_integral","config":"H","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_inequality","config":"H","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_substitution","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"diff_i_chain","config":"H","expected":"confirmed_error","predicted":"uncertain","fingerprint":null}, + {"case":"abs_incomplete","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_domain_sqrt","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_messy_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"ac8bfef7ed8c7604698f84532321cc417367a6f39c5d9c449df1aa8b1fc37d30"}, + {"case":"abs_lost_root","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_overlap_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"47d7aa89f19bf83ad6b1a1dc70dc553c1379fcb0c82797948cdb344d0ba7c61f"}, + {"case":"hard_messy_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_overlap_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_fraction_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"1e1488af4943af3c3ad28c02fe8ee23b5e741eef72bc1355f31135df6499f994"}, + {"case":"st_i_mul_2x3","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"hard_fraction_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"d7c9aab23af77e60f44face279dacc6d8fb39fe4e1307314c1ae3563e67b16bf"}, + {"case":"st_i_mul_3x4","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_v_mul_2x3","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_mul_3x4","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_two_errors","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"st_i_bigx_2x3","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_v_bigx_2x3","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_bigx_3x4","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_v_div","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_bigx_3x4","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_div","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c552db87e0f43b07e518c459317f3eb5306b5890b1f27f464d0cf184650765d1"}, + {"case":"st_i_add","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"eb06db0df854904167205a7daf662e7fd4aa3320012f0c59031342c71773a770"}, + {"case":"st_v_add","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_diff","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"st_i_diff_power","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"27fc1e0276480da656072b5157049c00d2a5279a4d8c9a3f3a594d18e4ed8c80"}, + {"case":"st_v_diff","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_dist","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_algebra","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_diff","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"demo_correct","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_expand","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_two_brackets","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_negative","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_like_terms","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_crossed_out","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_square","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_arith","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_factor","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_chain","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_equation","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_unconventional","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_quadratic","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_distribution","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e1141c11f1d72004b667721d625457f4bedf70061aee83121e30ab073bcee9bf"}, + {"case":"alg_i_square","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"5a2ebe1cd6298981aace9f4d01b84eb55ef82f379dbf737829f03befe791efd1"}, + {"case":"alg_i_sign","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"b3b93fb297a94023701b9c9ec6bc8655b354e5013c3699eab331ce8b135e00c3"}, + {"case":"alg_i_arith","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"58442325e681a7eb66ad433444459dd70d751a40d240eba3af1eb90321dcc642"}, + {"case":"alg_i_like_terms","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"0981a9217075da65f72d8244a5c881469ede06e30446538f37a2b630e83a05d8"}, + {"case":"alg_i_two_brackets","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"3a3fac44294f97ac1ee410a45acf9b102d238561f408ee18ddb439bdfad10bfa"}, + {"case":"alg_i_factor","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c735cb26eea690d7a3c52fae89fc558c3faa4ed7fdc10cb4e5c93360c3d8fd89"}, + {"case":"alg_i_equation_sign","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"18d193684972a86c8ddd857fff19d2182e3b3a3fddcb06472f44bca51e3d56b8"}, + {"case":"alg_i_equation_divide","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"824ce6da1f0844ac7f323307b23909dc3c26d1eb901128347bd6aaa76e27be72"}, + {"case":"diff_v_power","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_later_step","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"90e3f414eb263119efa29fffd90be12fe63b281b4c9935e8ed0eda4abaecb114"}, + {"case":"diff_v_sum","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_constant","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_leibniz","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_poly","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_function","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sin","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_i_coefficient","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"7ef1a56bbbe0fbe7adf87bd8ef561f8e7d06dda562b5a31977178227309a8e4f"}, + {"case":"diff_i_exponent","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e6ffbc72abc45894e8b6f38505d7f90486f297d9ba39717be4e422acd8929116"}, + {"case":"diff_v_chain","config":"I","expected":"no_error","predicted":"uncertain","fingerprint":null}, + {"case":"diff_i_leibniz","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a095ec1e895734cdb3ab5ec498bc4a7ab1f69449462c55ce92b77f9994cacfeb"}, + {"case":"diff_i_constant","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"4dcfc8cec5ae4ccc1f6faec2bb3352dd946d8e6f76d1847bf7bdd201e4881f9e"}, + {"case":"diff_i_missing","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"befe77023a414fae34dadba3da42915142362779cd01d5d274102d7a8fc1e833"}, + {"case":"diff_i_function","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"24cc3dea012470a6f45e7539700f9a0d8cea6e0d2ab94b25c38aec3e984c9320"}, + {"case":"diff_i_sin","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"6910e10d635fa5a411494361dd0eef03073af077a0c7f1ead135569ec10d433b"}, + {"case":"abs_integral","config":"I","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_inequality","config":"I","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"diff_i_chain","config":"I","expected":"confirmed_error","predicted":"uncertain","fingerprint":null}, + {"case":"abs_incomplete","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_domain_sqrt","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_substitution","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_lost_root","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_messy_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"ac8bfef7ed8c7604698f84532321cc417367a6f39c5d9c449df1aa8b1fc37d30"}, + {"case":"hard_overlap_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"47d7aa89f19bf83ad6b1a1dc70dc553c1379fcb0c82797948cdb344d0ba7c61f"}, + {"case":"hard_overlap_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_messy_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"1e1488af4943af3c3ad28c02fe8ee23b5e741eef72bc1355f31135df6499f994"}, + {"case":"hard_fraction_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_fraction_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"d7c9aab23af77e60f44face279dacc6d8fb39fe4e1307314c1ae3563e67b16bf"}, + {"case":"hard_two_errors","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"hard_crossed_out","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null} + ], + "v7-compact": [ + {"case":"st_v_mul_2x3","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_mul_3x4","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_mul_2x3","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_i_mul_3x4","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_i_bigx_3x4","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_v_bigx_2x3","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_bigx_2x3","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_v_bigx_3x4","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_div","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c552db87e0f43b07e518c459317f3eb5306b5890b1f27f464d0cf184650765d1"}, + {"case":"st_i_add","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"eb06db0df854904167205a7daf662e7fd4aa3320012f0c59031342c71773a770"}, + {"case":"st_v_add","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_div","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_diff","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"st_i_diff_power","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"27fc1e0276480da656072b5157049c00d2a5279a4d8c9a3f3a594d18e4ed8c80"}, + {"case":"st_v_diff","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_dist","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_algebra","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_correct","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"demo_diff","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"alg_v_expand","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_two_brackets","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_negative","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_square","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_like_terms","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_arith","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_chain","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_factor","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_unconventional","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_equation","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_quadratic","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_distribution","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e1141c11f1d72004b667721d625457f4bedf70061aee83121e30ab073bcee9bf"}, + {"case":"alg_i_square","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"5a2ebe1cd6298981aace9f4d01b84eb55ef82f379dbf737829f03befe791efd1"}, + {"case":"alg_i_arith","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"58442325e681a7eb66ad433444459dd70d751a40d240eba3af1eb90321dcc642"}, + {"case":"alg_i_like_terms","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"0981a9217075da65f72d8244a5c881469ede06e30446538f37a2b630e83a05d8"}, + {"case":"alg_i_sign","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"b3b93fb297a94023701b9c9ec6bc8655b354e5013c3699eab331ce8b135e00c3"}, + {"case":"alg_i_two_brackets","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"3a3fac44294f97ac1ee410a45acf9b102d238561f408ee18ddb439bdfad10bfa"}, + {"case":"alg_i_factor","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c735cb26eea690d7a3c52fae89fc558c3faa4ed7fdc10cb4e5c93360c3d8fd89"}, + {"case":"alg_i_equation_divide","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"824ce6da1f0844ac7f323307b23909dc3c26d1eb901128347bd6aaa76e27be72"}, + {"case":"alg_i_equation_sign","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"18d193684972a86c8ddd857fff19d2182e3b3a3fddcb06472f44bca51e3d56b8"}, + {"case":"diff_v_power","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sum","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_later_step","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"90e3f414eb263119efa29fffd90be12fe63b281b4c9935e8ed0eda4abaecb114"}, + {"case":"diff_v_poly","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_constant","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_leibniz","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_function","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sin","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_i_exponent","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e6ffbc72abc45894e8b6f38505d7f90486f297d9ba39717be4e422acd8929116"}, + {"case":"diff_i_coefficient","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"7ef1a56bbbe0fbe7adf87bd8ef561f8e7d06dda562b5a31977178227309a8e4f"}, + {"case":"diff_v_chain","config":"H","expected":"no_error","predicted":"uncertain","fingerprint":null}, + {"case":"diff_i_constant","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"4dcfc8cec5ae4ccc1f6faec2bb3352dd946d8e6f76d1847bf7bdd201e4881f9e"}, + {"case":"diff_i_function","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"24cc3dea012470a6f45e7539700f9a0d8cea6e0d2ab94b25c38aec3e984c9320"}, + {"case":"diff_i_leibniz","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a095ec1e895734cdb3ab5ec498bc4a7ab1f69449462c55ce92b77f9994cacfeb"}, + {"case":"diff_i_missing","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"befe77023a414fae34dadba3da42915142362779cd01d5d274102d7a8fc1e833"}, + {"case":"diff_i_chain","config":"H","expected":"confirmed_error","predicted":"uncertain","fingerprint":null}, + {"case":"abs_inequality","config":"H","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"diff_i_sin","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"6910e10d635fa5a411494361dd0eef03073af077a0c7f1ead135569ec10d433b"}, + {"case":"abs_integral","config":"H","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_incomplete","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_domain_sqrt","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_substitution","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_lost_root","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_overlap_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"47d7aa89f19bf83ad6b1a1dc70dc553c1379fcb0c82797948cdb344d0ba7c61f"}, + {"case":"hard_messy_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"ac8bfef7ed8c7604698f84532321cc417367a6f39c5d9c449df1aa8b1fc37d30"}, + {"case":"hard_overlap_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"1e1488af4943af3c3ad28c02fe8ee23b5e741eef72bc1355f31135df6499f994"}, + {"case":"hard_fraction_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_messy_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_mul_2x3","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"hard_two_errors","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"hard_fraction_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"d7c9aab23af77e60f44face279dacc6d8fb39fe4e1307314c1ae3563e67b16bf"}, + {"case":"st_v_mul_2x3","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_mul_3x4","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_v_mul_3x4","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_bigx_3x4","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_i_bigx_2x3","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_v_bigx_2x3","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_bigx_3x4","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_div","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c552db87e0f43b07e518c459317f3eb5306b5890b1f27f464d0cf184650765d1"}, + {"case":"st_v_div","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_add","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_add","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"eb06db0df854904167205a7daf662e7fd4aa3320012f0c59031342c71773a770"}, + {"case":"st_i_diff","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"st_v_diff","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_diff_power","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"27fc1e0276480da656072b5157049c00d2a5279a4d8c9a3f3a594d18e4ed8c80"}, + {"case":"st_i_dist","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_algebra","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_correct","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"demo_diff","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"alg_v_expand","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_like_terms","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_negative","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_two_brackets","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_square","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_arith","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_factor","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_crossed_out","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_chain","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_equation","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_unconventional","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_quadratic","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_distribution","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e1141c11f1d72004b667721d625457f4bedf70061aee83121e30ab073bcee9bf"}, + {"case":"alg_i_sign","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"b3b93fb297a94023701b9c9ec6bc8655b354e5013c3699eab331ce8b135e00c3"}, + {"case":"alg_i_square","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"5a2ebe1cd6298981aace9f4d01b84eb55ef82f379dbf737829f03befe791efd1"}, + {"case":"alg_i_arith","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"58442325e681a7eb66ad433444459dd70d751a40d240eba3af1eb90321dcc642"}, + {"case":"alg_i_two_brackets","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"3a3fac44294f97ac1ee410a45acf9b102d238561f408ee18ddb439bdfad10bfa"}, + {"case":"alg_i_like_terms","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"0981a9217075da65f72d8244a5c881469ede06e30446538f37a2b630e83a05d8"}, + {"case":"alg_i_factor","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c735cb26eea690d7a3c52fae89fc558c3faa4ed7fdc10cb4e5c93360c3d8fd89"}, + {"case":"alg_i_equation_sign","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"18d193684972a86c8ddd857fff19d2182e3b3a3fddcb06472f44bca51e3d56b8"}, + {"case":"alg_i_later_step","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"90e3f414eb263119efa29fffd90be12fe63b281b4c9935e8ed0eda4abaecb114"}, + {"case":"alg_i_equation_divide","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"824ce6da1f0844ac7f323307b23909dc3c26d1eb901128347bd6aaa76e27be72"}, + {"case":"diff_v_sum","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_power","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_constant","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_poly","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_leibniz","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_function","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sin","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_chain","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_i_exponent","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e6ffbc72abc45894e8b6f38505d7f90486f297d9ba39717be4e422acd8929116"}, + {"case":"diff_i_coefficient","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"7ef1a56bbbe0fbe7adf87bd8ef561f8e7d06dda562b5a31977178227309a8e4f"}, + {"case":"diff_i_constant","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"4dcfc8cec5ae4ccc1f6faec2bb3352dd946d8e6f76d1847bf7bdd201e4881f9e"}, + {"case":"diff_i_missing","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"befe77023a414fae34dadba3da42915142362779cd01d5d274102d7a8fc1e833"}, + {"case":"diff_i_leibniz","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a095ec1e895734cdb3ab5ec498bc4a7ab1f69449462c55ce92b77f9994cacfeb"}, + {"case":"diff_i_function","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"24cc3dea012470a6f45e7539700f9a0d8cea6e0d2ab94b25c38aec3e984c9320"}, + {"case":"diff_i_sin","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"6910e10d635fa5a411494361dd0eef03073af077a0c7f1ead135569ec10d433b"}, + {"case":"abs_inequality","config":"I","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"diff_i_chain","config":"I","expected":"confirmed_error","predicted":"uncertain","fingerprint":null}, + {"case":"abs_integral","config":"I","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_incomplete","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_lost_root","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_substitution","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_domain_sqrt","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_messy_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"ac8bfef7ed8c7604698f84532321cc417367a6f39c5d9c449df1aa8b1fc37d30"}, + {"case":"hard_messy_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_overlap_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"47d7aa89f19bf83ad6b1a1dc70dc553c1379fcb0c82797948cdb344d0ba7c61f"}, + {"case":"hard_super_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_fraction_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_overlap_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_fraction_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"d7c9aab23af77e60f44face279dacc6d8fb39fe4e1307314c1ae3563e67b16bf"}, + {"case":"hard_super_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"1e1488af4943af3c3ad28c02fe8ee23b5e741eef72bc1355f31135df6499f994"}, + {"case":"hard_two_errors","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"hard_crossed_out","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null} + ] + } +} diff --git a/services/api/tests/test_postgres_ledger.py b/services/api/tests/test_postgres_ledger.py index 475d6ba..97b03a5 100644 --- a/services/api/tests/test_postgres_ledger.py +++ b/services/api/tests/test_postgres_ledger.py @@ -1,14 +1,20 @@ """Run only against a disposable database named quack_ledger_test.""" +import asyncio import os from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from dataclasses import replace +import httpx import psycopg import pytest +import respx from quack.llm.ledger import BudgetExceeded, UsageRecord from quack.llm.postgres_ledger import PostgresLedger +from quack.llm.router import DEFAULT_ROUTES, MAX_ATTEMPTS, ModelError, ModelRouter, Purpose +from tests.test_release_safety import Answer, completion @pytest.fixture @@ -16,8 +22,9 @@ def url(): # type: ignore[no-untyped-def] value = os.environ.get("QUACK_LEDGER_TEST_DATABASE_URL", "") if not value: pytest.skip("set QUACK_LEDGER_TEST_DATABASE_URL to a disposable quack_ledger_test database") - if psycopg.conninfo.conninfo_to_dict(value).get("dbname") != "quack_ledger_test": - pytest.fail("refusing ledger tests against a database not named quack_ledger_test") + info = psycopg.conninfo.conninfo_to_dict(value) + if info.get("dbname") != "quack_ledger_test" or info.get("host") not in {"127.0.0.1", "localhost", "::1"}: + pytest.fail("ledger tests require a local disposable database named quack_ledger_test") ledger = PostgresLedger(value, 1, 1, 1, 1) ledger.database.execute("TRUNCATE quack_model_usage, quack_model_debits") ledger.database.close() @@ -130,3 +137,73 @@ def test_quota_and_usage_tables_have_no_public_rls_access(url): # type: ignore[ connection.execute("SET ROLE quack_ledger_public") for table in ("quack_model_debits", "quack_model_usage"): assert connection.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] == 0 + + +@respx.mock +@pytest.mark.parametrize("failure", ["timeout", "invalid_output"]) +async def test_router_retries_share_one_durable_operation_debit(url, settings, monkeypatch, failure): # type: ignore[no-untyped-def] + monkeypatch.setattr("quack.llm.router._retry_delay", lambda *_: 0) + first = httpx.ReadTimeout("slow") if failure == "timeout" else completion("not JSON") + route = respx.post("https://api.test/v1/chat/completions").mock(side_effect=[first, completion()]) + with ledger(url) as instance: + router = ModelRouter(settings, instance) + result = await router.complete_json( + Purpose.EXPLAIN, [], Answer, session_id="alice:s", + route=replace(DEFAULT_ROUTES[Purpose.EXPLAIN], retry_timeouts=True), + ) + assert result.value.answer == "ok" + assert route.call_count == 2 + debits = instance.database.query("SELECT * FROM quack_model_debits") + assert len(debits) == 1 + receipts = instance.database.query("SELECT * FROM quack_model_usage") + assert len(receipts) == 2 + assert {row["debit_id"] for row in receipts} == {debits[0]["id"]} + assert [row["record"]["attempt"] for row in receipts] == [1, 2] + # Missing/failed usage cannot refund any part of the original retry allowance. + assert instance.project_spend > sum(float(row["estimated_usd"]) for row in receipts) + + +@respx.mock +async def test_router_cancellation_records_receipt_and_retains_reserve_after_restart(url, settings): # type: ignore[no-untyped-def] + called = asyncio.Event() + + async def waiting(_): # type: ignore[no-untyped-def] + called.set() + await asyncio.Event().wait() + return completion() + + respx.post("https://api.test/v1/chat/completions").mock(side_effect=waiting) + with ledger(url) as instance: + task = asyncio.create_task(ModelRouter(settings, instance).complete_json(Purpose.HINT, [], Answer, session_id="alice:s")) + await asyncio.wait_for(called.wait(), timeout=5) + reserved = instance.project_spend + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + receipt = instance.database.one("SELECT record FROM quack_model_usage")["record"] + assert receipt["error_category"] == "cancelled" + assert receipt["estimated_usd"] == pytest.approx(reserved / MAX_ATTEMPTS) + with ledger(url) as restarted: + assert restarted.project_spend == pytest.approx(reserved) + assert restarted.session_spend("alice:rotated") == pytest.approx(reserved) + + +@respx.mock +async def test_failed_durable_reservation_never_contacts_provider(url, settings): # type: ignore[no-untyped-def] + route = respx.post("https://api.test/v1/chat/completions").mock(return_value=completion()) + with ledger(url) as instance: + instance.database.close() + with pytest.raises(ModelError, match="ledger"): + await ModelRouter(settings, instance).complete_json(Purpose.HINT, [], Answer) + assert route.call_count == 0 + + +@pytest.mark.parametrize("bad", [-1, float("nan"), float("inf"), True, "0.1", None]) +def test_invalid_receipt_preserves_debit_and_blocks_further_spending(url, bad): # type: ignore[no-untyped-def] + with ledger(url) as instance: + with pytest.raises(BudgetExceeded), instance.reserve("alice:s", .02): + instance.record(UsageRecord("test", "low", "transcribe", "test", estimated_usd=bad)) + assert instance.project_spend == pytest.approx(.06) + assert instance.database.one("SELECT COUNT(*) AS n FROM quack_model_usage")["n"] == 0 + with pytest.raises(BudgetExceeded), instance.reserve("bob:s", .001): + pytest.fail("corrupt receipt did not block spending") diff --git a/services/api/tests/test_release_safety.py b/services/api/tests/test_release_safety.py new file mode 100644 index 0000000..2a0cc07 --- /dev/null +++ b/services/api/tests/test_release_safety.py @@ -0,0 +1,376 @@ +"""Release boundaries: bad configuration, ledger integrity, retries and private diagnostics.""" + +from __future__ import annotations + +import asyncio +import io +import json +import logging +import subprocess +import sys +import time +from dataclasses import replace +from pathlib import Path +from unittest.mock import Mock, patch + +import httpx +import pytest +import respx +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pydantic import BaseModel, ValidationError + +from quack.config import Settings +from quack.llm.ledger import BudgetExceeded, Ledger, UsageRecord +from quack.llm.router import DEFAULT_ROUTES, ModelError, ModelRouter, Purpose +from quack.main import RateLimiter, build_services, create_app +from quack.observability import JsonFormatter, RequestDiagnostics, request_id +from quack.schemas import AnalyzeResponse, Mistake +from quack.services.hint_engine import _log_failure +from quack.services.verifier_pool import _child + + +@pytest.mark.parametrize("name", [ + "budget_project_usd", "budget_session_usd", "budget_request_usd", "budget_hourly_usd", + "reading_cache_size", "line_memory_size", "verify_timeout_s", "second_reading_patience_s", + "rate_limit_per_minute", "global_rate_limit_per_minute", +]) +@pytest.mark.parametrize("value", [-1, float("nan"), float("inf")]) +def test_invalid_limits_fail_at_configuration(name: str, value: float) -> None: + with pytest.raises(ValidationError): + Settings(_env_file=None, **{name: value}) + + +@pytest.mark.parametrize("name", ["budget_project_usd", "budget_session_usd", "budget_request_usd", "budget_hourly_usd"]) +def test_zero_budget_is_an_explicit_spending_stop(name: str) -> None: + assert getattr(Settings(_env_file=None, **{name: 0}), name) == 0 + + +@pytest.mark.parametrize("name", ["rate_limit_per_minute", "global_rate_limit_per_minute", "verify_timeout_s"]) +def test_zero_rate_or_timeout_is_rejected(name: str) -> None: + with pytest.raises(ValidationError): + Settings(_env_file=None, **{name: 0}) + + +@pytest.mark.parametrize("overrides", [ + {"max_tokens": 0}, {"max_tokens": -1}, {"max_tokens": True}, {"max_tokens": 1.5}, + {"timeout_s": 0}, {"timeout_s": float("nan")}, {"timeout_s": float("inf")}, + {"model": "unpriced"}, {"reasoning_effort": "typo"}, {"retry_timeouts": "false"}, +]) +def test_invalid_model_routes_fail_before_a_request(overrides: dict) -> None: + with pytest.raises(ValueError): + replace(DEFAULT_ROUTES[Purpose.HINT], **overrides) + + +def usage(cost: float, **fields) -> UsageRecord: + return UsageRecord("m", "low", "p", "r", estimated_usd=cost, **fields) + + +@pytest.mark.parametrize("bad", [-1, float("nan"), float("inf"), True, "0.1", None]) +def test_corrupt_ledger_amount_stops_spending_atomically(tmp_path: Path, bad: object) -> None: + path = tmp_path / "usage.jsonl" + ledger = Ledger(path, 100, 100, 100) + ledger.record(usage(1)) + with path.open("a") as handle: + handle.write(json.dumps({"estimated_usd": 2}) + "\n") + handle.write(json.dumps({"estimated_usd": bad}) + "\n") + for _ in range(2): + with pytest.raises(BudgetExceeded, match="ledger"): + ledger.check("s", .01) + assert ledger.project_spend == 1 # retries cannot count the valid prefix twice + + +@pytest.mark.parametrize("record", [[], None, 5, {"session_id": []}, {"timestamp": "yesterday"}]) +def test_invalid_ledger_structure_stops_spending(tmp_path: Path, record: object) -> None: + path = tmp_path / "usage.jsonl" + ledger = Ledger(path, 100, 100, 100) + path.write_text(json.dumps(record) + "\n") + with pytest.raises(BudgetExceeded, match="ledger"): + ledger.check("s", .01) + + +@pytest.mark.parametrize("bad", [-1, float("inf"), float("nan"), True]) +def test_invalid_reservations_cannot_increase_the_budget(bad: float) -> None: + ledger = Ledger(None, 1, 1, 1) + with pytest.raises(ValueError), ledger.reserve("s", bad): + pytest.fail("reservation accepted") + assert ledger._reserved_project == 0 + assert dict(ledger._reserved_session) == {} + + +def test_reservation_is_released_on_failure_and_cancellation() -> None: + ledger = Ledger(None, 1, 1, 1) + for error in (RuntimeError, asyncio.CancelledError): + with pytest.raises(error), ledger.reserve("s", .5): + raise error() + assert ledger._reserved_project == 0 + assert dict(ledger._reserved_session) == {} + ledger.check("s", 1) + + +def test_out_of_order_timestamps_do_not_pin_expired_hourly_spend() -> None: + ledger = Ledger(None, 100, 100, 100, hourly_limit=1) + ledger.record(usage(.1)) + ledger.record(usage(.95, timestamp=time.time() - 7200)) + ledger.check("s", .8) + assert ledger.project_spend == pytest.approx(1.05) + + +def test_failed_ledger_write_counts_call_and_blocks_further_spend(tmp_path: Path) -> None: + ledger = Ledger(tmp_path / "usage.jsonl", 100, 100, 100) + with patch.object(Path, "open", side_effect=OSError("disk full")): + ledger.record(usage(.3, session_id="s")) + assert ledger.project_spend == .3 + assert ledger.session_spend("s") == .3 + with pytest.raises(BudgetExceeded, match="ledger"): + ledger.check("s", .01) + + +class Answer(BaseModel): + answer: str + + +def completion(content: str = '{"answer":"ok"}') -> httpx.Response: + return httpx.Response(200, json={ + "id": "c", "object": "chat.completion", "created": 0, "model": "muse-spark-1.3", + "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}], + # Deliberately no usage: charge the conservative fallback. + }) + + +@respx.mock +@pytest.mark.parametrize("failure", ["timeout", "invalid_output"]) +async def test_retry_must_fit_remaining_budget(settings: Settings, monkeypatch: pytest.MonkeyPatch, failure: str) -> None: + monkeypatch.setattr("quack.llm.router._retry_delay", lambda *_: 0) + ledger = Ledger(None, .06, 10, 10) + router = ModelRouter(settings, ledger) + response = httpx.ReadTimeout("slow") if failure == "timeout" else completion("not JSON") + route = respx.post("https://api.test/v1/chat/completions").mock(side_effect=[response, completion()]) + with pytest.raises(ModelError) as caught: + await router.complete_json( + Purpose.EXPLAIN, [], Answer, session_id="s", + route=replace(DEFAULT_ROUTES[Purpose.EXPLAIN], retry_timeouts=True), + ) + assert caught.value.category == "budget_exceeded" + assert route.call_count == 1 + assert ledger.project_spend > 0 + assert ledger._reserved_project == 0 + + +@respx.mock +async def test_model_cancellation_releases_reservation(settings: Settings) -> None: + ledger = Ledger(None, 1, 1, 1) + router = ModelRouter(settings, ledger) + called = asyncio.Event() + + async def waiting(_: httpx.Request) -> httpx.Response: + called.set() + await asyncio.Event().wait() + return completion() + + respx.post("https://api.test/v1/chat/completions").mock(side_effect=waiting) + task = asyncio.create_task(router.complete_json(Purpose.HINT, [], Answer, session_id="s")) + await called.wait() + assert ledger._reserved_project > 0 + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert ledger._reserved_project == 0 + assert dict(ledger._reserved_session) == {} + assert ledger.project_spend > 0 # cancellation cannot erase potentially billable work + + +def test_rate_windows_expire_at_the_boundary(monkeypatch: pytest.MonkeyPatch) -> None: + now = 100. + monkeypatch.setattr("quack.main.time.monotonic", lambda: now) + limiter = RateLimiter(1, 2) + assert limiter.allow("a") + assert not limiter.allow("a") + assert limiter.allow("b") + assert not limiter.allow("c") + now += 59.999 + assert not limiter.allow("a") + now += .001 + assert limiter.allow("a") + assert limiter.allow("c") + + +@pytest.fixture +def client(settings: Settings) -> TestClient: + return TestClient(create_app(build_services(settings))) + + +@pytest.fixture +def log_output(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> io.StringIO: + output = io.StringIO() + handler = logging.StreamHandler(output) + handler.setFormatter(JsonFormatter()) + monkeypatch.setattr(logging.getLogger("quack"), "handlers", [handler]) + return output + + +@pytest.mark.parametrize("path, method, expected", [ + ("/api/health", "GET", 200), ("/api/not-real/private-id", "GET", 404), + ("/api/analyze", "POST", 422), ("/api/health", "POST", 405), +]) +def test_api_responses_have_private_correlation_headers(client: TestClient, path: str, method: str, expected: int) -> None: + response = client.request(method, path, headers={"x-request-id": "forged-id"}) + assert response.status_code == expected + assert len(response.headers["x-request-id"]) == 32 + assert response.headers["x-request-id"] != "forged-id" + assert response.headers["cache-control"] == "no-store" + assert request_id.get() is None + + +@pytest.mark.parametrize("origin", ["http://[broken", "ftp://10.0.0.1", "http://name@10.0.0.1", "null"]) +def test_malformed_or_untrusted_origin_is_refused(client: TestClient, origin: str) -> None: + response = client.post("/api/analyze", json={}, headers={"origin": origin, "x-forwarded-host": "10.0.0.1"}) + assert response.status_code == 403 + assert response.headers["x-request-id"] + + +def test_request_logs_never_include_private_url_or_body(client: TestClient, log_output: io.StringIO) -> None: + response = client.post( + "/api/analyze?secret=private-query", json={"image": "private-image"}, + headers={"authorization": "Bearer private-token"}, + ) + events = [json.loads(line) for line in log_output.getvalue().splitlines() if line.startswith("{")] + event = next(item for item in events if item["event"] == "http_request") + assert event["request_id"] == response.headers["x-request-id"] + assert event["route"] == "/api/analyze" + assert event["duration_ms"] >= 0 + assert event["status"] == 422 + assert "private-" not in json.dumps(events) + + +def test_unhandled_exception_has_safe_response_and_diagnostic(log_output: io.StringIO) -> None: + app = FastAPI() + app.add_middleware(RequestDiagnostics) + + @app.get("/api/private/{user_id}") + def broken(user_id: str) -> None: + raise RuntimeError("credential-and-handwriting") + + response = TestClient(app).get("/api/private/user-secret") + assert response.status_code == 500 + assert response.json() == {"detail": "internal server error"} + assert response.headers["x-request-id"] + output = log_output.getvalue() + events = [json.loads(line) for line in output.splitlines()] + failure = next(event for event in events if event["event"] == "request_failed") + assert failure["error_type"] == "RuntimeError" + assert failure["frames"][-1]["function"] == "broken" + assert failure["frames"][-1]["line"] > 0 + assert failure["request_id"] == response.headers["x-request-id"] + assert "credential-and-handwriting" not in output + assert "user-secret" not in output + assert request_id.get() is None + + +async def test_concurrent_requests_keep_distinct_log_contexts() -> None: + app = FastAPI() + app.add_middleware(RequestDiagnostics) + both = asyncio.Event() + ids: list[str | None] = [] + + @app.get("/api/check") + async def check() -> dict: + current = request_id.get() + ids.append(current) + if len(ids) == 2: + both.set() + await asyncio.wait_for(both.wait(), timeout=2) + assert request_id.get() == current + return {"id": current} + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://test") as client: + replies = await asyncio.gather(client.get("/api/check"), client.get("/api/check")) + assert len(set(ids)) == 2 + assert all(reply.json()["id"] == reply.headers["x-request-id"] for reply in replies) + assert request_id.get() is None + + +def test_json_log_format_includes_timestamp_and_structured_fields() -> None: + record = logging.makeLogRecord({"name": "quack.test", "msg": "test_event", "levelname": "INFO", "count": 3}) + result = json.loads(JsonFormatter().format(record)) + assert result["event"] == "test_event" + assert result["count"] == 3 + assert result["timestamp"].endswith("+00:00") + + +def mistake(**overrides) -> Mistake: + return Mistake(**{ + "id": "m1", "category": "arithmetic", "error_type": "arithmetic_error", + "operation_type": "arithmetic", "previous_step": "2 * 3", "current_step": "5", + "verified": True, "is_new": True, **overrides, + }) + + +@pytest.mark.parametrize("fields", [ + {"intervention": {"shouldQuack": True}}, + {"mistakes": [mistake(verified=False)]}, + {"mistakes": [mistake(), mistake()]}, + {"mistake": mistake()}, + {"mistakes": [mistake()], "mistake": mistake(verified=False)}, + {"mistakes": [mistake()], "mistake": mistake(current_step="different evidence")}, + {"mistakes": [mistake()], "resolvedMistakeIds": ["m1"]}, + {"mistakes": [mistake()], "status": "no_error", "intervention": {"shouldQuack": True}}, + {"mistakes": [mistake()], "stale": True, "intervention": {"shouldQuack": True}}, + {"mistakes": [mistake(is_new=False)], "intervention": {"shouldQuack": True}}, +]) +def test_unsafe_alert_contract_is_rejected(fields: dict) -> None: + with pytest.raises(ValidationError): + AnalyzeResponse.model_validate({"snapshotVersion": 1, "status": "confirmed_error", **fields}) + + +def test_valid_alert_and_silent_acknowledgement_pass_the_contract() -> None: + alert = AnalyzeResponse( + snapshot_version=1, status="confirmed_error", mistakes=[mistake()], mistake=mistake(), + intervention={"shouldQuack": True}, + ) + assert alert.intervention.should_quack + acknowledged = AnalyzeResponse( + snapshot_version=2, status="confirmed_error", mistakes=[mistake(is_new=False)], + ) + assert not acknowledged.intervention.should_quack + + +def test_alert_validation_remains_active_with_python_assertions_disabled() -> None: + result = subprocess.run([ + sys.executable, "-O", "-c", + "from quack.schemas import AnalyzeResponse\n" + "from pydantic import ValidationError\n" + "try:\n" + " AnalyzeResponse(snapshot_version=1, status='no_error', intervention={'shouldQuack': True})\n" + "except ValidationError:\n" + " raise SystemExit(0)\n" + "raise SystemExit('unsafe alert passed optimized Python')\n", + ], capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stderr + + +def test_cors_exposes_server_generated_request_id(client: TestClient, settings: Settings) -> None: + response = client.get("/api/health", headers={"origin": settings.cors_origin_list[0]}) + assert response.headers["access-control-allow-origin"] == settings.cors_origin_list[0] + assert "x-request-id" in response.headers["access-control-expose-headers"] + assert len(response.headers["x-request-id"]) == 32 + + +def test_background_failures_never_log_private_expressions(log_output: io.StringIO) -> None: + def broken() -> None: + raise ValueError("private-student-expression") + + connection = Mock() + with patch("quack.services.verifier_pool._limit_resources"): + _child(connection, broken) + assert connection.send.call_args_list[-1].args == (("error", "ValueError"),) + assert "private-student-expression" not in str(connection.mock_calls) + connection.close.assert_called_once() + task = Mock() + task.cancelled.return_value = False + task.exception.return_value = ValueError("private-student-expression") + _log_failure(task) + event = json.loads(log_output.getvalue().splitlines()[-1]) + assert event["event"] == "hint_preparation_failed" + assert event["error_type"] == "ValueError" + assert "private-student-expression" not in log_output.getvalue() diff --git a/services/api/tests/test_routing_regression.py b/services/api/tests/test_routing_regression.py index b6354a9..fbc3882 100644 --- a/services/api/tests/test_routing_regression.py +++ b/services/api/tests/test_routing_regression.py @@ -15,12 +15,11 @@ from quack.llm.ledger import Ledger from quack.llm.router import ModelRouter, Purpose -RESULTS = Path(__file__).resolve().parents[3] / "evals" / "results" +RESULTS = Path(__file__).parent / "fixtures" / "recorded-routing.json" def recorded(run: str, config: str | None = None) -> list[dict]: - with (RESULTS / run / "results.jsonl").open() as handle: - rows = [json.loads(line) for line in handle if line.strip()] + rows = json.loads(RESULTS.read_text())["runs"][run] return [row for row in rows if config is None or row["config"] == config]