From abdea25f7acd4c23511bf319b5969ee3615e0601 Mon Sep 17 00:00:00 2001 From: Brendan-Lim <121050815+Brendan-Lim@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:24:32 +0800 Subject: [PATCH 1/2] Integrate production-compatible release safeguards and offline CI --- .dockerignore | 4 + .github/workflows/verify.yml | 86 ++++ .../fonts/bricolage-grotesque-latin-700.woff2 | Bin 0 -> 22400 bytes .../app/fonts/figtree-latin-variable.woff2 | Bin 0 -> 20184 bytes apps/web/app/layout.tsx | 7 +- apps/web/lib/persist.ts | 36 +- apps/web/lib/scheduler.ts | 14 +- apps/web/lib/types.ts | 1 + apps/web/package-lock.json | 18 +- apps/web/package.json | 10 +- apps/web/playwright.config.ts | 3 +- apps/web/playwright.production.config.ts | 6 +- apps/web/tests/e2e/accounts.spec.ts | 1 + apps/web/tests/production/safety.spec.ts | 114 ++++++ apps/web/tests/unit/release-safety.test.ts | 228 +++++++++++ apps/web/vitest.config.mts | 2 +- docs/RELEASE_SAFETY.md | 118 ++++++ package.json | 8 + scripts/check_publication.py | 1 + scripts/check_release.py | 43 ++ scripts/public-files.txt | 13 + scripts/verify.py | 72 ++++ services/api/pyproject.toml | 2 +- services/api/quack/account_api.py | 13 +- services/api/quack/config.py | 25 +- services/api/quack/llm/ledger.py | 52 ++- services/api/quack/llm/postgres_ledger.py | 26 +- services/api/quack/llm/router.py | 212 +++++----- services/api/quack/main.py | 22 +- services/api/quack/observability.py | 88 ++++ services/api/quack/schemas.py | 25 +- services/api/quack/services/analysis.py | 2 +- services/api/quack/services/handwriting.py | 11 +- services/api/quack/services/hint_engine.py | 2 +- services/api/quack/services/math_verifier.py | 24 +- services/api/quack/services/practice.py | 3 +- .../api/quack/services/starter_questions.py | 10 +- services/api/quack/services/verifier_pool.py | 5 +- services/api/requirements-dev.txt | 1 + services/api/requirements-tooling.lock | 20 + services/api/tests/accounts_support.py | 3 + services/api/tests/conftest.py | 18 + .../api/tests/fixtures/recorded-routing.json | 371 +++++++++++++++++ services/api/tests/test_postgres_ledger.py | 81 +++- services/api/tests/test_release_safety.py | 376 ++++++++++++++++++ services/api/tests/test_routing_regression.py | 5 +- 46 files changed, 1999 insertions(+), 183 deletions(-) create mode 100644 .github/workflows/verify.yml create mode 100644 apps/web/app/fonts/bricolage-grotesque-latin-700.woff2 create mode 100644 apps/web/app/fonts/figtree-latin-variable.woff2 create mode 100644 apps/web/tests/production/safety.spec.ts create mode 100644 apps/web/tests/unit/release-safety.test.ts create mode 100644 docs/RELEASE_SAFETY.md create mode 100644 package.json create mode 100644 scripts/check_release.py create mode 100644 scripts/verify.py create mode 100644 services/api/quack/observability.py create mode 100644 services/api/requirements-tooling.lock create mode 100644 services/api/tests/fixtures/recorded-routing.json create mode 100644 services/api/tests/test_release_safety.py diff --git a/.dockerignore b/.dockerignore index f9327cc..03bb1f1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,6 +11,7 @@ !apps/web/public/ !services/api/quack/ !apps/web/app/collect/ +!apps/web/app/fonts/ !apps/web/components/account/ !apps/web/components/ask/ !apps/web/components/collect/ @@ -29,6 +30,8 @@ !apps/web/.gitignore !apps/web/app/collect/page.tsx !apps/web/app/favicon.ico +!apps/web/app/fonts/bricolage-grotesque-latin-700.woff2 +!apps/web/app/fonts/figtree-latin-variable.woff2 !apps/web/app/globals.css !apps/web/app/layout.tsx !apps/web/app/page.tsx @@ -121,6 +124,7 @@ !services/api/quack/llm/report.py !services/api/quack/llm/router.py !services/api/quack/main.py +!services/api/quack/observability.py !services/api/quack/prompts/__init__.py !services/api/quack/prompts/ask.v1.md !services/api/quack/prompts/explain.v1.md diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..02ebe36 --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,86 @@ +name: Release checks + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: verify-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 45 + services: + postgres: + image: postgres:18 + env: + POSTGRES_USER: quack_test + POSTGRES_PASSWORD: test-only-password + POSTGRES_DB: quack_ledger_test + ports: ["5432:5432"] + options: >- + --health-cmd "pg_isready -U quack_test -d quack_ledger_test" + --health-interval 5s --health-timeout 5s --health-retries 10 + env: + CI: "true" + QUACK_ENV: production + QUACK_AUTH_MODE: "off" + MODEL_API_KEY: "" + QUACK_LEDGER_TEST_DATABASE_URL: postgresql://quack_test:test-only-password@127.0.0.1:5432/quack_ledger_test + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: | + services/api/requirements.lock + services/api/requirements-tooling.lock + services/api/requirements*.txt + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: apps/web/package-lock.json + - name: Install backend + run: | + python -m venv services/api/.venv + services/api/.venv/bin/python -m pip install -r services/api/requirements.lock -r services/api/requirements-tooling.lock -r services/api/requirements-dev.txt + - name: Install web + working-directory: apps/web + run: npm ci + - name: Audit locked Python dependencies + run: services/api/.venv/bin/python -m pip_audit --no-deps --disable-pip -r services/api/requirements.lock -r services/api/requirements-tooling.lock + - name: Audit web dependencies + working-directory: apps/web + run: npm audit + - name: Install test browser + working-directory: apps/web + run: npx playwright install --with-deps chromium + - name: Install checksum-verified secret scanner + run: | + mkdir -p "$RUNNER_TEMP/quack-tools" + curl --fail --silent --show-error --location https://github.com/gitleaks/gitleaks/releases/download/v8.30.1/gitleaks_8.30.1_linux_x64.tar.gz -o "$RUNNER_TEMP/quack-tools/gitleaks.tar.gz" + echo "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb $RUNNER_TEMP/quack-tools/gitleaks.tar.gz" | sha256sum --check + tar -xzf "$RUNNER_TEMP/quack-tools/gitleaks.tar.gz" -C "$RUNNER_TEMP/quack-tools" gitleaks + echo "$RUNNER_TEMP/quack-tools" >> "$GITHUB_PATH" + - name: Run release checks + run: npm run verify:release + - name: Upload browser failure evidence + if: failure() + uses: actions/upload-artifact@v4 + with: + name: browser-test-evidence + path: | + apps/web/test-results/ + apps/web/playwright-report/ + retention-days: 7 diff --git a/apps/web/app/fonts/bricolage-grotesque-latin-700.woff2 b/apps/web/app/fonts/bricolage-grotesque-latin-700.woff2 new file mode 100644 index 0000000000000000000000000000000000000000..9e04f022a4cf051ac828b67c914d82c6e9de960a GIT binary patch literal 22400 zcmV)0K+eB+Pew8T0RR9109Swj4*&oF0KPZ?09POY0RR9100000000000000000000 z0000Qff5^mwQd}ZQU+i@Q&d4zRse-25eN!_ks$Or3xpZ~0X7081BfsLAO(j22Z<6J zs;n27n>vbi?zRC0uh(GLC2BUp_BaG{kv}skY#aaz@?7@+{|O0l$gl+UpmeRy--`&5 z!m3oN(ZkC+t!qwwTB|3`P-$7Jv=-`3%#oJX?eXG@gRa|a3C=-Fj%P+9D44d8vE~i`)ge1IpMi1FZ=gzPg@4} zRQbTbvWzm4Q-8s`L_i1}+~=JE?yw?8SDZx8`}h1;-Rtc8A&=x3MhIpzp^yYp5QC8% z{dwDm=XLYXeJk@N5RyQU8AK3JaKml^EjXAd5v$DVMri<1fiN8#$HI*hI}#Aluh@^2 zsB8ydhaHRI!j}K(REB6d8zO$X=a%!F;x03Q$*7lIxy084c2~~s8Yx)kU1hgYZ3jU)h7P#H zL+8@jVHkeiETsgg;W>IYm}OoVgs$E}(GC(Wb=N zC^&C!MQ$u(t zfrApA5)U16C_}1J>Qd)il`dUw={lt<7G3XLyEIg|k@_+!hhb5Vd|_--1FKrx#UZrq zU;DMX9}!E$PrECT(kStrVq%N$9Z4;bG!hx6i!beS>4s#IEYZZ9Rc#PT)alv?snD%K zg=lgp-J$?2d$7oZNB7crEilzBjw6#Zj3D7EU-~wF+Lu&}iB*)*jLRrho7kV&ZtwT~ zM3?DL4|F(mAGZvph=_zBL4pJcQrzd?{gXNWqA$JX| z@OBg{gcric97L35h!r*<4mf}~UClrUX>0zP(f&HTz#u!HLv1?a~Q#Erk6p6QzV(>#sSFy^@e&f^r0c6NsQXV+|@amj3&u4$Qt>*{1o*+iTZ z7}r>!N0t?U;%OHTZx}-w%)n_OO;P`Lw6FWDzx2C)c`WRChtF`9xbgeF+4_Fg%dPH- z9&RPr@lva(v3=0Dx4HGLY6^Rb0(qtV#J-P_~PO4F|oY&mT3!y&6_Td2QXQa+PS>D z8txUvPc^e^9UDZN77TucgLcf-65!F;>d=PY!~-Uj)n{5go!;P07Fjsi2}CQp>mAWS zn$y4^{Gl~{Tsi2=-c4Q=fEDCO%IKM#~Mw%+3)wcC8qk+|2ojO+{Zzu#`ZV> zKjHrXv%xA>F_DRy)n}d=5fL>DXgYKdyq#IfA}?{26(iZnj;(C1?_i(a>E%`GpxwlY zCF2wkuR9^C5YxC!Y;$WD^Ax{Z|PNJPbSl+zZ?V+>EeCtN>vstOthF zO}pPe(c)^~5ZCQq!hv%L(GPvj+DM9^z}cwv4hGhX9ymZKaSAkKyi|uqjU;Kc$Irl) zLw#dOF87UEcJ-{QY?MSYO)K>0z5}0kEJJ?>+_pp;1=<|*t6zMn zqZ_9TKE@ly>ee&;2qB$NMNI(gjKs}t`V4*9TM4iWaW3zO`CIk<68+1c{4&GFSolJFUzYLR7L9ry% zgze)v5yTq_j-X32BHmCSDF-Brps>hA%0fOTQaY(jP$c$i*qEr3ssd-b5u8IcU77`O zHe)Fupg5~l>Zl+G4AOx`?HQp{-pK8mg*kcC!G0Fx zC2`UzPUcoI>UG)D7HVl~*VF^V2zUbC-qwRm zQ>YR|=&%s5MvYKX9sx_|A&JW7K~^IW5CBnwMFo`PQr4&@l~Ui0Z~`$2K?$oMnl+1G z3aMkcz9KZJioM20E^^&Ss4vB`APp|Eh{dAONI7PQ%Pg3%qy@+%@VJ+E5k2V#HRS(P zTwNxInouGm0M&s?P=ie^sM`uD#18DC{TPH|$`i_?G2&v^#fBhJ&>~MC5&)PPl%r2F$q0 zrqip!&gF#drGiA-UQ~!B2uTBxjIFY?p7s8Q>D8*V5qDV4MgkZwW@qAB6343;qXX|>#cZefW_s*QDAsjra$eQx|4nMvn>OmxdgF!-yf!Yi3AG}cb~`BW=t*OZ(Hq_5 z`d%9Ed06SIS}s*DdT#flyS=bg`mk6C=yi2Laa(?n=%6Nsl&|ZFE$-?!tPv?*2Bb48 zgi_6bnq`-mdUE4ON!noyOvt2&41}CoMv}hM6NNX%qhDh%Syrs0FynaafO;FjrUr1B zx8jX6H&u1r*^K0lg}2q}-K=1-g>S7kkzgNS0(6y>wFon;MfD$5gQB!j4LFk&f+m6x zA#H|0EfS@oOpcRHMZ}OSfGZA0oC(_|F_J_?K^J7nmt#TzVK=}qF^=79=Ltlvvnz{SW<`y)hL+qhA$cD{)DbO6Oy0ksVuC>;{ ztQHA@AdOXkpT?S5RT&Q0!05c(o~SGqW%-Vtn?&ucR4$536mx(fwwpxacFmgDF+tW% zvCNe8+9<#vQ~g}8#%ZtpuKhH(Q{t*yFB;Wu1|^4r>XeJ9D?A1J#!*uwRbEm^yETGT)mN%}h!M7rl5QcDDhXF3s3M-N z0vd?gYRluBgY;C{3f`Ad@UJPA46lZtZSTO%X@R`3Nm=Yv2nXy+NFBZf`MTyckBpA6 zBRE357$0PtgCTh82!!0M+Ok#m!is|TZIYlkB1qP<{U(RjUEgHN39tEif*4b)lEjW@ zTI1x3nb|u56nSCKtPbOhHG8x$C2ogGo}e-krHfx$gh5srD*L5Gu9Ug@@Y@Fr*j`?% zmW>XxKt(AAbK;a#uuH1ohHFVO!X-lSIMKGqcNSqkL$QpCV^Vudqd|`bLW$#@k8nlW z^F~j(2^sO>*;J-<(`+oj01Zs-&YP3>gp)7ADy?pn!-Xztf zb&8VUEl&$JuSQwZuFaP|Y_c0xoC^Q?M~%L|PYt!fj!ZuMwrC0Pr zWvI{;cSo6)uu}>LNcL9r7IX;~ykTq{(-d|3;DSy7SeWq$FM7!q$Ik&1t3di|pPONc z(9MhmLnK`#n~Oy#2SZCL3%+R_;-jJ!BSg<=wR&LlW04*5?r5~VpP?ql*I8V9*5g>2 zCK?`t|I2Z?&-)6m`LnksYhU(d4FM0o;sr{`1~zAHA*>$j(xU%o0$*ml+-bX2zU_V``7JkS2tIY$=$Befxziqb@vWO7(gEFr zfVQ%cSj*lCd+Gv1SE00Ti}~m+W*~XEqyg1NPn$^pzpv@Q!x?dcrfSeFI zZ`i0At=8;56w!9am8~}cC`ywc3qWB*%4)RVdAeUZ>y)mcb12YeOR9^rlvHNQrD(NE ztd4q?V9LENj^`CLOMshN05{Qt-ihv>cF~N1eVS}_C=~iyEp}2Hla$rw5S$8GYrISV zOC*p&07CLZGMI#-NJXg{YzZr*f)KLhr4Ue9VS5h`oUvmM&ngk@*_H2Ew7tCfYnV&Q ze~3|~%6qCks~t;~yaMd5#!Z@1K(Q&m0b>Si@V)BFo!DIi zJS<07>xoQS0*cKo4>)KAdzF+%aB)9LkNfe1n#cpQXbHWd@OjJOJJ_PR*o=+S7V(;q zG$rii4mMSDYO_veZSu!s&jNYRtXy#HHgFz^CS5O%{emQ5RSvRbl)T_LGW5X@%)GKf%h(}?PjlwLGf ziXG}i!2}HbxBqaBh;#wqV!|3DL#9ep7l_a}AA(HQg14#-6snS<6^#6QV0#QCCe*>G z+y~qII_Rb9^R=X?K7^DM>&wXlOc8e+Ap~oPwJ@2nmx3Vc&*ZwP`cP|!$oo;+sz#SiWggvBtAD~mp=^HS?Fq87qhrTlU z=t#sQEA)`wn!AS0D`=WqnCCv^_Q}Ze>X{5C^qKSfrLX($m(&u?ZW^tQt{eui$9wM8 zEk%0|LfWH*9x5g3$*H$KbQ7KJUMaoeAL+__tx7bg7o~R7W=(v=vtB6DDh->4KHH6G z=}vKHc-zyUg)Tkq;EeY@Q!OQX4}zdA^5{1J0uwRsedsge*|wShCxDcq!+rWwHH0dv z!tN!25jC$#_9E>~9eK1jPx7RCArP7+W20Ey1PzX)k8+~kF)d_uCr>m8! z&V)nUMJ7ToH7sIW*O?MJYhjmGg$)E^*I`2@NC(;MA^CL%QXiBNbh{YHV|QwHRtWCh(K(BTm+z{| z&BKH%CL}3NWU`%khgRDrq|{(E3A)>D)C|kjuym}nUR=Ue8}|b&{id}wH+h+j|L8zp zz^@e8SJ5&h)+{`LL!ISoO;IP_ZNr~otG!+Jv{S!Tj|H1!7Jb$Val5`-oA6)W{=wu% z)+r~GdM|_?lu`n&P6O0+sKA?e!y>@Dqg{RZKmKqnlX(0$I()968 z5}h;2J_LpR+F0nvCKs?Pk!CRcUo~kE5h={nGwfd>Kkd;-8(7-N3xFI-cj;S|r|&G@BtkL)w19$==9M(NZ(in`#b<%>eDWzrAL=?N z)Pb43DF<(uHGqQ*Zur3uXYH$X(7slC6=t~hL3PioEc(Py zFb2-kW}5ldJ%Xc)C^iW7GPuV8Co-T;F#n-k-rLzBeoj>S2JyjS#cQ}JB2jF0g!oIZ zb%k-K@~(>|%VsDRt?CyD%*1?0dLvppy~j1PPELeqRnnZJ*#`yLj;g4mRikLzeF|!Z z3#ggY`?-4$JX#T;zYtXGY=IG)jsNQ8dQ|Q4Zptj-KE~Ny{cu(~xTgrJd@0G8R$2-- zwkU@1OYc@WaK{IKlCqr69#HS>I~vJgLMUKy2k#4_NhN?c)3^|1P2g+a_=H9-T~5X~ z{`clIHKhGeYat!l-2$5$(V~QJH}-ZkUz+1JQPIKUjwzfCnG7ChYji}Sshk+Gr0nU; zOG!-wDk(QT5 zbdjL2*O0bd_OxB6HIJtw2OJ}zI%@+@>%j&+X=n>5wy-^DVJlhXCM)=+he`M#!Wr-I zvTX&&CHRzIcrg{JOyev!(Yt7da+-3N_^$^VB6;meWr|0ySGZ;<2`*dTx6}qc`u!+@ zwEa1n*DHiE;bE1sS$*MjGKmQaY&U&tntqu!uGfyOkaI^W{3I7d<>J>IE*h%ksCmws z)5#cnLet4;_JpU6DpaAHqEAjE&5rgb{FSehzATGEm3XqehnxTT9Qq3Uto@_E(0IVb znQVM4xME}($C&nbHeVUD)VweD8pyWxNnva*CEG{z;p-y;KCWSazQESXPbN`OhZ|*J zKg;%Pp%|`b^(gU$5wFqM4B3uDqU9*&Fy(M1UM5^UNq*<{c}f}ZmJd(pN?1N;Kfq2> zx(09o+-U(JZDug_#Yrsyugj!Lf1zEgB7N2&e#dU*NM5=+O{rH**R6&<_`@K{6qrqH znP*4>Hs!g|bQHn&Qq%IXZO8~IOYi<3G#&Iz*a61zV#h}UPw&%ei3$T=sx)la5d<-UemNcp z>KX}0L{Z0E}bd=anrBnVXW?mz!q70UR=0 zfbXRb08tb&xTs_Thz0-vlK{*o1ptJgp#hk~4!{?H9{^xH?x)R=6X>w@8UefkxV8td z7bXr*r)31-15Bi~fXD!f4_iVhJI5R}304v?T!`b0a=K9lBt`h*3^h1M36MO15GkD6 z)&hrS2YDC@L97l!yjv53BM|_ktOLNk$YkH1wP|+cME1}OWE0_v0UH348y!~r{$OZX^WiciA5i+cgL9#@4c!3A(OoC+tzVQ_HlMeJVe z7MrU$>NM&gYBOpCegiDhy!qPw4470o^o9SG@AXkiyD#Y7Q=Dp5Y;*uH$ zCT4l+T$k^b+lutwEcx4Exi$}N^Vk!WzVp391`IjkxnWgCj5tl>QkO25scFW$sU^HV`Dz&C`bWdkL>gtj1?t8%6LvoK;drYaP$9h}gYylvqnm)pi zkW(cut{|(hR8byd*NQ_WNu^n3y;5GNBFWS`bI;WmX=v(ZLSt<$RV;P1cCDj@&H`O! zy36(C>dj@vAvCM0g1EwzqH4xKNnB|{S)r;T$4i`Pth!uFoTa(e9K4uf`iPW-ya*$p zEKn77qTHDZEisme)+#R>(}=00FO`yHh{}n}izqBsRK!?MNtv=Xm6xc9I98#mfzuVv zG*X=vx6evTPodsIFB*FpHQs`>m?*VvRSX$9fxMW4Duqc!bPtzS3@8z$qi9((<#BHH zu~_l;=@h*(WTC=D23))kg5w7;F%yxVj*KkYG(-Q8-eaCZ{Ts2-YyB4`iQxoG(Jhj4NxvlBYEn6z}*22TX@Vtx*FsG(3{RO1)QMe!hgXL=)(ENn$6X(9&Y2mYG{| zwPn}ab6c3EmfdMtbITrAcv90qmxkhA78>py8>B6wR;a4c2#Fzt#hth&luh3XGwQp! znN*)kfm;Is(e|-l6>%n^I^$AYTb{=_!ho)aplDaY2xPO=HDbfnY%%RqOVqHZt!xwv zYhZpIv48}_;S)`yGf7hYjQX-uOnv8SKK%9GxYMtiMSI)(xWFXpDIac!4N<@HC=&}v z z6*Y1-W7;O_K1qZsQXo;8|6ZwwZ@zipx6RVtpK+rDofvEn{FhlNS5o4vX9D(8da`Frn*!o(XSxcyxpK*q8TRN^vPv40#QluCmynQ^XgZx>Nwf;0>RE~7_vPE>YinP7 z4KEu~A*BSR9Flyo0s}#zNWg5se83vE3~UtQ7y-qAQX*xc3Sy=dbEU>o78IHf-ZH>- zHYjSq^%~?zh9pa#rOMJ~`Bs`yY-WsqbCK1es_7}E1jfW*&uQ*T8* zPFLUSmeOMi5f3MwI+xo^o|Zf6SZVAXOK6~y1nEXlxq{(M|3kG|Q#?dFD&52A9q~6J zU4c6_ZVH$@(;=|8mVI^YFL9tPhq`d2wxf*@=Q`W<1l7R$TuhJ z6)Yt>dZg3ZmVedLgYrgx*MaZbH?W0+g``GM5d4lcys#dlgN_+A9?Hc#P|6fxE?a~T z2bZ`-8#~yA9?*u0F*-`EM`+^~K7#Bt8W|dBp@nsn&nLa1q!O5~9EG>cWrFMu3>~Zl?AEDiQkt`vC_42LXowhXF?bM31G4gy0Xn4Vcfs4~{;YF~BE~q@pkBUYSZ%q4kT_AIC!0Feh^{!BpSRV`T`F`rs;1cO&`{dT zxZ!}<#-vRNrLc&AlBnLIG=#*fgvrzf#?W;GXG+|S>(eBN5P&Fmru&p65?c%?02Bg> zK+{`Gz|AN}1xzqcTt~)&bq}XiDT0MTNKE2c$v$Tvd17ee&RP1f{G@1-inV zHIqMMBLR-7hs-UhV$oHAED)?TXNb!50jNva*pWeKLAd$ry_; zK0qu+EJLhBkY0I02CN)HxfmNcxr|rGT0L70JXp-sYO)UQI=SfLuA2*)cP<+dfJi_r zAc;LaA{npv6!rtlMb9tVqG}h#CmA-(h!IOECA@p=>dxYYXG?by4n>( zCAMl(8ysy+)ZONU%UCbf#Z115lkyPFw@H_iTzr^7ht;;F5FRKf(!`Wh6o%w9HL)xl zLz*eWoKlOm7`oDRn=(+6nS&nkQosc8ZsR^j08M~9fM!4opbO9qAk0Kqk)S88QuAgh z!PltcU`kJc-aMM(+sJAt%^8!5z-%uU%YbE-6=o|9Dza_``U=@#dWr1`VNXf& zsPc&(ej&7x@UG$FYOJ8+DZM#K6sRHf0AW*7A0nu4yeKadwFQ&pLj5g3S}5Qm|sl{a`WVV_N@>kvQ)W+oY~NG>Xus4j;b zS|V;l+==)$k@Pe68p0>MrcO0>{dq##wKbNH2kEW}#J5XzJ5i5S7K91Yn9 zjV`b}Lhhr+nDRiNTF4i(4b94?I)uOqKpQk)K^6*)NONZ8V&bpIJP7BD!g$WGgP>tTCMJ_;hEy}b_)8TiL4*WxQY2C;k-%b-kGZ8 zXo!*dU@ZX10crpa0JH!m1Mmgl2Vf?ESpfV2?Eo=XA$PGE^bC#Y#S_l1HWC>NCFN8R zkq>pU2#{5?TgEi11!N6udrat4O+uzjF2E^?7`J3efv`YkXP)D%asbB%DIZKZ5dhudi@L+~+%FWd8k_6okn67f2hc*y-2VGIcXsUt!C z8~^GLi`EQWSttH{*M+ZmMOHR8ymIZjEExU|bw(lXjq5VkV6Gn#`R2SV147>#itv9Y z$f=1^1g6mBOd8ZoTtrMt#)^WHnjL!%oVd^$ZvqIdrycILYF2m|Rva^W3?2`T=Htq7 zqFcJ5MaNg2x*N)P{)b2S4BL=~nT9y-DLxrnb~Hk0KkA>y^8>+Ugzz~>-;T06RCz!T>v$i}s!t11jwxgd|O z9b6IVPZ$Xx*RLI27B-_zMJBBsT{8SUiju!hJ!%x8FZoq^Fa*ppy51elJWtQlt4GJ z2s>BvO@`w-PYoU@(?aZSXz>y41q#`WVW zLR4ckfl{D?i=fy#mFMw+#1YNy(fjwf*&e`CDJf@jVlu19DaR8mR^u8T4xTeG@(t3JE zLvTm*n2zh)Sm-CZ5Ii${Bg%_SO%C)tj%64M)BzX(;PEe-pMW!5ZbirLJ+p)3W2poI zdI5D1+>A%MwY=P$=6j*u^46S91i*VvdjLD({E$3UXNRTHsXn1Ei7&tVAjN>QLQ>JJ@-O`K14Dx37(u177iW-6%8E& zi;$Rvl#HBOCXFmwAGxj7Z%SndVrF6E;N{~NP^egmQe`UCIxvKcP-r}XNTOQET|c;I z*vGE<$*=nT=&QlkHhSQ-Z^ro6E0@)b@VhVF80(Tv9=hPXi>}MSp)iD40wPj2kWkRb z2#84FT=B4R2yp%IpI*OY6qK@wG|8ir%Sg|_#LC0T#jSfADynLd`c51=a^~2#OdJNr zk`WZ!pnri{w>BL*wF`|Pd(1^3kplE;$P?l655RS=f%zBoM}PyhiEVbD6FO<7Xbjh1 zEc19Y^*^3ms55B+?JMfQ?p6!44e_d!v6V-v3G+G`u7W48t`_W41FL8PFDOr-TbU=;cmULLVuD(N~fpVqEw_bDn z_PI2dEgMBvD~2~;^nj|*q3J5g!IpodTP_(=zkEBbR=j6E-eK{fE6%To{6=1>wP=UkL+?^A&1TNY33u(;rry{>!CzMJUu3zRUMd_j z*$7i5xXxr<{fTiuAY zXgG&}JvMnnPN;X z%SBRlA7=teB;OPWq@;RLdq45)>O*cew@&{sk7tb7RqWJ+C zT|>H5msCM<;F-%XInIi@UE=#h?gzI<(R8v>FqFpZ(E^pq-uBK)W_@**1DeziAtE3m(2$;UgI0>X(9GO)wzIDowjN zKZ93ZY9$Eh8{ut~&=<>7s-O*UnRirxNEvw zEBD52t4T2DzjDL;h}D!4zOdm}cF^P`x^zPC<=z$njuSaZs8YzUAjooFHFEu;UWjpqqZQP1Y!EL?(BRIz$I=a zq}z@5yR| z#auokRQu?y<5x(?hr$147rAydp5I-pkF!(kbu(6jSOIxfS6}IwMS6aZ&XR^#uB^Nq z_?nb_eZ@TM@)#VIliujx@D)#aKJllkTBQKA&~+$6Q(< ziMCIIY`Re8cK6Ke$h`|Xt=d?8JIF0hLrk%YImRVjfwNUG$gGy`QL{-i61d2d;D;gR z;CKOg^(^-C#iV5H?v?>7R@DrNb`>RxE8g>x6?g(sGlU+ni^1oEIJ(8)eX)jIw8xcd zD=>Z~2A@QQQoDY%BE^o29FXG5<2B2_M<4w?(koBi3)?i!p&>9`dH-h*U08Ou+f_21 zTCBtKlx5%K!{F#rZ&qW~^)vk``@+~1*a&V2h{q8j8i0JFgZOYW)ruo)>wfW}>>&I7Am-IM48t}M>+it0$ z?UNn1RMN4tHP};bqhGVm+wnYO9%%vJ+1wek?$yG#7d|ms$5jWAJ{|#`0^T`{knjQi zeg{p2Umw~B@6y;!Ss3x;io-@vU=;RryW_*Hg(yf5Mp8ZK1u?|io~14{o8pfc8V65& zdBCP=B9$*fQ3BSLH+7bY3rKXBEBdZ1pv9TAiHAgqPu1ET?~L#RM{+DVvm$pW4Pr{u z*zG^@UC5u%qVmC0#g=iIf+QST4Oeo5CZtYIL{=8sgc3Xgov2(INxeD% zCrL>m8c8EIl|3crQY{dCYFHn(v2}Cd?RLj>JnimDxWVDeDv=mN>mJ?q1!4dwTP)BbQ*T|&wtbLVLC}<0zE6A z$QaqesCiLj7Nz97^SZzLkC3mUYNNbCeM3ufE;YuRMmE^HN}|IY(~=$y(<@?P#Pp+2 z7?cnEjCkeTOGX-fzy#n@lt?*i77kmHHJp3I2eC5r%rh!-Say~7KE@#|v+0M!fa6DU z7Sqbo&dr9vY;<$ENV{jZzlo!EZKqs;wK5ap{Xf%kF-0qQxfw(p-`(!hz^ZB~rmxlZY zl{E9;w(!mkrQ#Fh`uvfs+T$i89`!|ZbooRk-e^2vOmoi6r1$%eWSPh~c)}`{N*^a= zXOViBg2xtBMLo1Wxx#rbALI(QO=tVt-PpN}_$FvhZtw+c$hkm^!HqS)cRB7QW(lc1 zMXHH?Jx)z`-=hcprsd3Rm33d@?m!b88mPNGfv2pN7k1d3corH~)a;$xyF?;k0>=uO z?kh}Zb5DP?u6>6S1cV@4UHUqOG`zC!$k3#QN!DHM^MOW7*ztwe1K`SGjHDby&HRWNzM`qq2sk2UhZW7;DquF@HJI75;Nk+CxVVPg!Ifca^g8`UQZfA@CTOtWPKTU|CRDpN z%{ZieGa7in3x|u;Z#%SP(kR}KPP>f{I=LhGiTh&VW5`2h(?= zT5R6mdDUfy<23U?TM+wD)HlL@qu3=DBfc|mzk>8FJ2bqXNu9Web=W9Ct3XxLWJU{K62Wa=% zpR3#XKQW2p)pgav@$Q!H^K*OWgt#VMQ%+J>a~E*Vp>r!SmUNCu%1^eidk_@~MFC5; z*3-=^+|#++a5-J(mHNj^ntcwTP-x^?hCwgP+b0Z%MZ@`F@p!nTQ)1VYQ_O0f7c6B7 z?MIo^<##Cs`k=HnC{=1cBiRy%F>%zgJWB%?tnGe3%PHrUTe9$h?`%Z*k?sS1o4oev zIuE;5=k?2qvC`zZ$~hp~r)*SC^M$9#IX5#W>E6=48Q;{!Mx(JIuK3={~la7XPBFw)1rGe^Z@7F?A(Q4_l+=$MUs}B4I~Ob zgoOL24+NzxHQ9mW@!-?M=M)*N{)~bn2foUx@)kr)FCnB3%%sQ&Q_byT^?r5zJ(5=Y zEY9eT{#E_YFqdU@Jwq9V zcv#>1XsR#c6bzNl6$sQ;jvlXdYiGM*V~5+NDG+LkQmhzlaW)aZYHorDiGo5*v%0WA zat_%u*i%tW`m62$!kV7Ugd69J6RAp*p5BnBXV;l>Ft0Rwn%xP8af9dIXy4?pYb`pH z-nO%Kuc4mm&~l4(UhBe@>iYX{%r%qoe0A$83wbwX1xA8s(FOq>&-HXB|1L0@UN z(pR%8=-Sj&t{5o`j>){Ka9t$P5B$kLOIxEwu;L(%(4Js0I5*N(I>?5?j@|x}xrk{BV!lSR@{2({uHH7JtbHc=AaX_T%}5B#%() z(CG4tN95qrT;*Jd)Q{^2&9K}Gf>kOt;qnzTGm}b&$w*hKOmv1xIk|3TV*OgD-@8Ao zacDnZJ8*Iv5w=~k?fNW^&fTzTLTV@BwTDyM6W_3L_TyGUG9TXn=qJ}L$NZ+p+~$93 z?pVL~&hK6je}aHVU4NcAy33rfCgRBNEnkAU&AU9!-2cs7#9(T$AG{3{n^|0AhA8>3 zCHU9>-loD(dPUS&{>TXL^-muLA-T6oE~^iguw(3T4)$nPf&q!Xl-ck1}DA1wm^d-gs|ZeekZ63sI=GmPL~Ncr0a`uqcX(s%gb5#HTAM+rwQ zHeo-(nIs4r%iL&!*y@Nt=cBa6JJ!Ycf-!c(4Whr7$!ef^v$7e^^;}}^a)f+{5_N6$ zU}f3mEQ8}7;{Nmd5fG*xD#PENA+t-nE*euR#5CqC7=P(wBL?wfgt?FpTB{Mx76{?c4keEo>gyl z*QOf`73o^n-lan6(X$ll#!r5pkV$bZ7Yd$Mr4N&B4D*X&3IJWE8nb$^ARrqox7w6e zuGn!B-+HtVdu00Kn7+-JLlUQr4Z4{(ZuZzh(7fh$JAs2I+hlTzJrhV|c{Ox%E~`}M z%4jOp>I4Cr-g*?DM|LZ{`PQyX9}E{POkFXx&SnjZ%F&f_1EbDttLIv~D0~0-jh;ZD z#9eaW@TSFMP)EGz7so5YfwI8@GWRQ~(>yDNhh9 zlPUzn`V+w98lTyZ5j8K)#GMy9%>q?f>DV&t%(JjOkRiw;22g(h6Kt9XX4pEuBOLb< z;#w2lPfs0?E|$Qj6)J-3sSO8ApRV0~ckEaDCkK9&drXExi5+E1EhJAeeOGYa9q)hT z>Rc!nTc$9yDnJGhF{iqTc*nV7KLBNGHkfd#)p#`}Bq=7F*@ zOMjov>+j3843w4I2KsXBjeIl7q0&%1eIjm%>`)h&3x$3u|6gpo+o%fF=SBC`@`A8P zQd3*5DX7%YUp_NzG9OiI`9Eq@ZOP}Vb0T6*fY1_a+7shvE^z;k$JJ6is(?AHBj|gc z%7-b{+C~>asxD^bSlbcLO}B>loQ3%IdsML$Z6xd^Vn@D{%H)uVzXuo(9ZBs4GsvZ4 zyPYkw)ml_T1%BB;h25^OaK)D2pps2t;@of=2T!s~6=a)+!?^KFj~xVUj@<45#$5-_ zt_M&gcCgB1URfN)GOTH8RF9h&i05RI9T{?x4YyYKMemoVc<#_FF?^^L^R>ikcy$4EDI8cVR;$m7q$DDAqB7#}KnC_|sa$EV z6*!F5EWP1q7f~&*4(1=wU*CIaH1>;Y@fG@~@-p~g}UhBMtVOgY`%^9ZgnMY5e;rsV<6)WgGpLO%yqY=8v=Q}H}CNg7v~GuCaFdltQLW| z)P{8TyX zau?W#{zdGwaIW>E)XZl1o5xajt=)or(6$^;1vBGm8>6>Egdwoo*oq8i;lxB;$zbjdDwcP$Eb`D0c#^14NuAWpce_jp)RmIg_V7q zDO`8|QOE54+kEQPPjEQ>V5pwDR!8}qYqW8%HDYrJ7`3p07D_<;6Qzj>L6uLI5~1WH zy~Ukq?0s+^UeHw4V3ij|gxQ6SRSmX`qQZ1l0rmrUx)>dYgT74@idk{M1?`{|A14%3FE1t?UZ|f=VjI~psAEuMXr}4w5`rPohq^6P# zjWMDTMXh0DV^0RCM9ftkJJ4z0zRX`H4`P2ww~Z`k*-B!Fye_r@8Is`~&XjnqbDIUt z)Rc*{icRT6pUzm#^L0|*`{^G%<8~6ARTj%4EC1;OeP+sJ_GHPD2g#w>o(p}4Jv~{I zjO`&{M-&Y6t)W~Q&tyqIoivIY#U`JQ0LXc8jcg?1K6EgYr74<7nlU@(R{1s-3AP#@B3b#7A{3{tr&jD3=B99+g~ zTL0H9boxmhDkMd5?95aw)3M!xW~fRfStUjR(Wf<5a;?$jLEDbjcKfDf?wS^c=UX-> z2}L9w|MVX`{k?}!^ST;|HNqVUcGx!e`+R{Ozj1R*K5xtssTOP6@rP8H0==8Iu!o0z z&4Md5>M#+_C*Wbrq%qS7gDOYns{1H?fPDaUXxH($bRzk+FptCe{(JeP190ja2eXTP z-C|RbQO@wJvt+PdtWF+{>jpC3TthTwu^nTy+cH)r z=_H1Up&Q`Nh{Va3b#=PGr6~)$%+SjS#+bYi0b2w;bOj*_} zTH?q`-k^NczC~p{=1Gp7Ou4U%Zy=baALfL8xPBizcaPoOGH3@8vMxk~R)$W@xz?I7bxKDZJ<|VA+ z@GFn3`5_7-Btv7=%;Zdlqxn2dULaA$zVJScv6ySF7z-L52g{~WcX12l=TlnF)emg( z{G(1<9#X3NNQnOTl%y z+DD#XccEDO@Z?D%`jc`bYI}1h@+VdGxx`4yzLi{l3X!Oy+Zy61BE%k0?MX@B8 zIkIiu)sBOox5e73_;4;hHD>c-Ec*yYfcw3~Ld-6Pj^2giexl&BNQk>t@bJ~lh1Pm~ z!Z|jotB8usoG`9j3T|g)g$aKOB>p9ds- zN8np!i@r;PJzj}`Z9E50ySnPXa$?ekKrUm)=f6(yR%-Kj&gDtqvkcbI;SZhd1UIZM z1^r*?HA*U9lmzwYu}ZH_hr82gBq^*VDoK^ettu}QseU1nafpk%6sYI!A@M(e)|Io9 zD`!Dxn(V$%bYCXFFA9O!C5~`^OhCTWf~X>yE`!<&-1qi#-V@z%Ys>y0RI4rS*!x4f zD?`GESNVZZcpo%e@aAZaHZ7&_zT@FV*pbA6V#&(qSUC|OA!cpU_+l0t#&HamlsD`1fuozdEG4(xZTar>dlL93 zZz&gedfzQP;=#zLl(d;!@yO(PPN_)5DLyy3k5c^jF{Sjb z;@tteG$)501>BMK7ydYvTs$-R!D}#v*V~^2dm<8sjL2O1oyaFkQBT0nOIGbuw%2Jl zMr*Yj+UqN&ap8VtM}5wQ+Gx(kjygwc=xSPQtMdJPEiD!>?4X z2mtG)V_ElC9mHTSVcoQSLp*kB_GkNYw?5_5>ORJ!2wPmORD-d?jNm9O@QyX@S$c?)UOBTU@h7qAqp zhZ|AkYt0X+xC3a+{h4De-!FWTf@wfk*mL-b#?_+=KTBfCkf$5}u|j*<8WElL8J|Hq zDNn%u@_Id}*jfz;M)9x3fbmIY1iEAoHi_;0 z`(oKMdh@ut@M-{aULay_Jdu?;@J1M~P+DqPc2hMo&vdMl@BWL$l>AC1BIoRnJeaXd zB8qYUHrU(8lq8MDzKE3vT=cQ##vFtuEM9POr^TrOYVTn=)!DvMeeWF2-VmM2a}Jl610^|Hw=T~mARuJ4##OZr@(dXok-<*l7LdQ9kbk3)o{_r% zD2`(PU7GbTaO_eIsSXb2A@d-LHN*n{&c?4^O<3JF8&L5kG<}vjp4q{y*4+cbcJEm# zk{vyRHXsiz=L+>_16I7cS52k-RA0F5^3xgWQzOQDmv)#9u06 ziv`+}Qbu$M#U`azSI{8~1-qq&FmZ%M73uX%e$5A1(u+>)>zgA4SGv@q$<`MPN&q(l zZ&^vOY+5O3usP}XJrBB@EMp~GY`jahpLz3^dJ`5bbmj^K$2g*T>OI)L-5AnCJpK-y zoQ3Af@CKv4Pmr4vT9|Iu7E(8o?ZoAT)Qp{+^z9t7?8_4hgMfo^{u1DAQszG&OaI5L z^76hNyIy$UJvb4?;T^yy!yV(MC}{?Qjk2vWv8nbL)Az`Hpx$1JE5#lveZO>5>3gMx zqECKltXY|xuqx!x&u_JZ&%1Uc4vCxAZf^nEZ$`I|g4I2Npv%2^`Q2xnKje0GBfogZ zc?pxXp2oaL#!~tU#u5f=_J)?qF9F#r$F`4cyW)psN3LftMm(}^-gfLYaLJ;8>*kJ$ z1D_LWyJ`~$TDG@r%j(_U3ugavw4|93%}8jFUuH~l3DYU2R0E;6FYTS%{{MD(XdeR<2y@8uPnfhBN@aVu z8mY0GRV0H8JX!g`RSQ##I7!QfTn zt^4h3ED6T6*WC#Nu^mVNeFHhZjYrX;1dj`QE~V66cxQJtNmLO;6~zMZhEAi=#Tky< zCk-vS6qf=hv~5jL;N6sLXb_%6+6eA02sq1M5o2r1VgI;h+xWj-vY&F)@$vt9fnDA> z-6BZ{K__J7>Ty3YFg>%_lAAfoR)i3b211XGu;*!agl-f zm{%-6)6ViUuUUR(O~#+*c&UPDx84l{Cx5@a#^TY}BslC}H}^+Cpf#O)VaJyA+v~((4;AJS^Vp@ySNt zH2cllRF>HMZdMjPKp?cdMdb<1r z{?3hYX@3Xx{(w7wUJj#hnSYsooBx_`THek$A;6fKUDdTnhc>Y|dG!R;9Jns%W&rl&$WnR*Qx{Wrf3fH_fX3ufEIF zy5x6!gWr1=&Q_(hGzn(o0EbnXB;=qU7{Q~*uK&^8Nu1YP*5j-Fn)_ClSF%fwTc2X? zB)hY^)4?A)AROZIJDjT~W^82nO}%9rR@&wML%*cAkM1|VuFiVuX(5C_E(F1c!G*yk z^xktz8gW|kHMgWq?`=Dmh&}EX+!>Um?WWSUiMG(>a{w{96{}{BTUgmCY!&%JE7(fF zw$NpNNtarvf^CK>&)BoNOW=6i3*31cNJVR@5>Lu{kptS7Wx(!2Ep9>e9_fIiTb&p@ z&LvzJG^b|ggs}h2Z4ALD3fTOOuvP=Be3pWKnu@zoUqzpn)5bPmB)(0ubz8EsbCi3$ zFZLQ`Ytq`=&MEM!ZyiI*CaJ#1gZ|69{xtUvZ$FEp?*+=O4xZo#?{TK@8{J#`*=zb+ z^EVXk{n3T2``}n3|LwE(ekTM(`1ATTtCZ{hn?%!tz%MSC_Z5KO{wi&F?2lW0kY*ae zp#Xu`W2YZAiJh6*7;*AGob#!DqWdPrIVRudEPCD)h_~y5{u7@r77YZ&IUTqlI$Rin zIs5PaJbX>W@+3dcF)Q)v_q-{Y8qGta_ z3p7@E(6M=m4784VW!n#2e3>G-z{- zO%73XizzAL+)`exVB0o!t>N35QTQKqr>mG0yM;F^X!4xN`GAoPZ1Xm4-mM!%(x%z_(kBbdb6gHxmAqF!oKH*s1Kt7PkJMT)MmTti z?HlT_O-l_uK$X3$-1*pf+Fv26fg+2x$kI*R$H9JNsyLO%Y1rlI5keksR&+#JLY)YC zB1I>UO^AH9h`pE6vC>KDSmg_1!^O9PK zXNg5N+VXzE8DX7JBM~J6IUkugdXAd_nby0|XSH{T06(fMhltE4D8e+Or6h_G6C75O zCY&f6N@k|;L4d1yA&_!HK};E8m<3ZqGLKZ)=+gqvX%b*O+A)fm4NKI7oJBFjfe=EA zigt~h$^t;BS_cVYDDW_*P;iQegj!&bXo?}k!~~knHz1ruv7byLio(Ra+ zz3H{WhX5X4K0iXl0J*&*4B9!X1T5)W6c#_ucp@85IOkI?nS!*$BF}=vB^~m@jJH98 zjS3XnO-^*ljXrtyh_Kt2zV(%_ z{j*~cKH;GZ=NB0gklSpnlAY~lkV&D`B>rUgAzk$|tqi=fyA&+vz6Pti`Lo}Sm8AcQQfYn*V&uFQy>FeEJ9w20K`t0(r+Xrd9V~Jv+_WU$z6y#D@RnYA;#pFnI%i#QWT5dVRo%jNuhz?T0z^w1;%%QTsV!erl zaZb4&j(BJkEHsDbmBzDD@uprcQr$F5(@XKX{wMwK^+u0IonM7VDwVlLw57Uplcx=L z=C>7|MxXA`CeG;l`5f9)^#=3Kb{4?Rkwr-E-7oj_ar@1j2pJT}hO*ZvH4{@6T4nk| zZAZNt@Q;6Yo`e&hK@9I9yH|<528yBzb zzjq$Mk{E>%MpZVGD&;ATe0*%!dsXzHY zkxy+ICQ<~f21KL?%|B}MRsBYwKTXYiKr?V~iekHsOTbqWJ+Ln-{iYTikZxU9_sk_f z)F3H^xX%-kiIYSf8Odw}TF^SdtK{0MxFzH-7%=n33;(ESo}nPwpS;=Y zKS52b217)6f9YDKlJ2xiE~2<@@nKwqu#HzNg~h+THxF?gR3Wj+1ajh*1iiyKqdo>T zR!iEyEgJAs3Q!uV(3UeQ%(D3CV1MO@9Q*9e_D@cEaZVl;=XBC{!X4fRG{*8QR;Yc*+C<*jx;N z3@{1;;4p^nicGS%#DdVvPv&)9yKn>?6i@(S-E>HaJ3fGjMZpWJx_(Qxh&Y{VjJ6OE06cRy zis-nFdy2L?Q4I=xoW})-*;Q_quy+uF_eb{7BMe5z527&#j2zkr2<{>Z910kvJ8I%- zxG?4${PUb;+Jj-&7!U)6md5>7NrRYz&N2Ji?l`Se9;Uf-(S-2~*e*M_NU-ypSIzTP z^I&klfnM`9&Np;IL%~3FCG!_Ba)%0w20?2XC8vgFbuL=WU&EdH4Q00B8N6sM5U@=w z29#z2mWD`ha>{LVlmy_c%&t{*m$*H}VJ%v^^J+{^C`OkalkwWc$3}{$fqwHH_22`~ zeErceTwM&vEWc53{kOdU=@}o@JB04b><;=?4Pvx)zKz%+lOt`e#+O)8T`Y)jn79&V z56=ib>AjL6T)N1&xZ`|PwoyJM47Yn5XftOG`TsPacV1wCJ%z{3cZPcJSyRea;7a-S zu_Sg-sm<&pc}FpqHxsFF{=Mh#43Q1wahKO=yj(%ef0)ZieU*;uR}8vVA}J=+I=5`^{e}u?c(7n2a=K8c*jI zirtcK6n&^@N#NbsTibUSddM*>hBmhng53_*8>gmZekmVS;63fy>_nI;>%(2e+MBWH zt#}#?12NIcE7iMInu*hJy~a)Vl5qo{Q1?0 zz;U^kzX4^9c4MHw_`rI+fH7^naAI^0z>??Xm!Zk#Dudi{Ol%h<2;8+bRzoKnk3KpT zEp46abhIDi=4#UBaq7Gh?52d29#{C?F3M*j+y(eeGDZE|5g5Yo~R<#`KoYk6+XVqU-&n2i74DzITB6(dQf3679UG;PXj z2J1%+O?SvOwG_~3atX(wG65d30U+fibeN*FcZeE?s&JVKcAbSOe7`ju`H3fQHSxc@ z&!t7GO>{7w(T2kI^>GHX~M<--a5<+WLc{!6kwFdd- z&2k0)1zuL8vUv$5u0Vi2`^^YkQ^Pap)mIPnn{E~FP3Rw+UsG1MyEnZHzT);|=#ZN0 zPcdW~LE~2sAua0c+ErjbjbQWgl!hs1pF*j3ObF-tvt~zinK1CWy{JaW6ll3c?=h#- zu!T(>K?f^5P1#yroqL{D<#`HJF)Nw}yc=Eqoup7+B8xlX=)k6(mRu3RbpkPG} zA%~MNg-BuhfhFSRE-l?J29KQDdwjzn6-G-FO&mj&NVye=FX<-$l+yw}ETzmBf*+TE zTzbWG41lTy)Tdbo630`ppoNhmNSZ*Uj2bkqRDvzoV@bw1G<-LyuV_fMIEFc9G*NJbM5?$XAV{Y6Qax5GiQj;0cf*QZ8?PFz@H$Eh{sW zLCZ>_`@gSP$@mw|IEt~9xO1xy2t^uSGIlhe#15zn=R&5TD~cXYn4wdka^VV*R@Im~ zYoI8;4?zMIGjaeaf|MbIYehT5q}3rLKO~KemNj(pz$_8YL>EDlM4epufJL!5zqrb| zqFas!rUK*umaJ|FeRRrvdl;G8I;$j539`te?U8uK+bIC^zY%?tG>Ab$kX|0TAy|!3 z!jfm^eN?Y;#;V4ikA_{9r?t&V6%;=Ez!FeI{eK*DZ)LjA#v7dspZNb__rgfsq$Pc< z1-*vB`(56zDbEZ!e<=3u&0NKI7sL90yHTRc1;CZRoLH*yzrl0~ytzRj!%fHaKV8MzJD_Z#f7+B2z5n$onXgP4;{&&U7=dUm%9McEt)S_jP0LPfq z>8zdc5YVA%F%*kdGQ4#y}!d;z6K)9p1SoniOl)q~{bDP|y;)Zf!4;#H}1& z(DurgI8jvT5@PA?2>HslxskYXA<3xkED_Bii4+IVdEe!clx-vx?B;KQ&plCaQLG8n zp`OX+PSJ#jL48m@C$}jfr7@>V31QkK6HviCdiA-|d6!(EWk8tLWb8j#}YX~?BfdQuo zX&9Jp8Wl$O1au(mb2B*)y9)%{EsV^aRrm1?31x4H9Mia!XT^)s&Ed?81xutsh}dB0 zAMU?G+^k@8L{lMByNaOe_+_{*(+(Duj#f`Iyjr&KPPkB9gHc^yRe#lwzy;kgTk~i$ zSTxFzR+`y>gJl!=S3IhAnF2{1tiLum3V=dlUPp!l1yz(`=a>o3eLsHVg#w1k!%{^W z^7uv4m=w7u<~L*a*|5_^FNC^mn)7fotB{4^)fFacL08B@(J zT!D9+c z2kTL}@ahgnCmL3PnmnmkGzkU?VMYQJV+Lg?aynRmg0V+q1`=dy*65=Oy5hD&=C;IzEF|2a9n)LpkcL zdUYr0BC{a#0f6KOz{O*;4$sDEGhD9Rkpv-N-*O&ys)zEV%z`YvnpGoLhhG-1%h?IF z<-pqjk~V?L&oi|ZvW!imwngwiVMyv2A=~7IVkx zRA0CA^ct#KNiN;sOq;i#%reY%Oqog^P8SFq08Y%8G}8^D@32j6sqX>FK93Qq9mrp8 zOU&QQ`B++o1pwC?WAgzV$2}V0x28S+m}RaZd+``Dt0bjWD8$Y_B=kZ_AZL_c%Cctm zb#gOyw%ff`S2xexRsrwX}vw;aDJ4;uTO49cFnZK$HkPR3B@4=IS!_R7V`PwTZU^k$I zpB?c2sV;BryZh(t-ETf@%~vXxRBZ2F0|00m$mR+k8Q|P8(Ol5{N{~w*LnI6 z$7P)?>u0}t?)y?1+vgS=h$&&Uvj1R^4dL>3j15fYXZl@+k&Fe80K zBir=kB!NT|tZW3x?0zbNRJ%665pu5KN~J=rXkKSQQhsxLkkhU$T~0UEnwa@U>)MdJ zG>q6y$lqIDv7hWxa9lFA(wJV5V&Jx>xNuw=uq3&(%G5zj^L#BM`w`xquTco8pEay9 zcql{6g%s3R3$73Vq~_6_3wC@5NaYU*2>1#dV91`2@H$FyGteO&&8nTb>}*0SP5on8 zQ5&LCWcfv4Ngda~VTH30E=h@~x?HuSQ-b`nyHJmH3A$V){$TlfHrAmqE7dUotcM4M zM1{M$zS83>UK_Q49{p+m98|R+cMYIXpa{VtM41udr2uofq_7_?lq0qrYQ0#3ffP$Y zZ#oL(Qfk9iNH)C!j{&ixJAzmpV|LtXk!B=P-lu2$HaS>Y_iQ$8%73O{E5McRZa|t+ zGv%WH2o@V{$I-X-ddpV%hU^F^4c>0^@Rn_{^D&u(QCQ5rsADHemR9*S%8^|wA5+I-hAiU2VBatDL~6D{#Pg+g$u z$K#W<%KLvnUS@!=8>QrZ!Bd&nIav$TQ}akxM!&60EQxmT zyT)LpSYM3&H_7U}$=B0Z?UjHm{9RXnhz-$z*@A?){y4hB-uIDH>fX^v%?1YiIjrSB z`<%W6G$iVYbOx-HYwpMl&0Hsla;0f2koC-m%3|!#8xLcEc&p@sW88>LTxvwzGi{W; z18Y=Z@`oixz5Y@9p!5A3pmabxy@Rk$8o@CDRq0IYuirB)h*Gp zqbo~bh{T#|rLrnzOR}c7_cxQ(ZmG*#*r*HIfpNrU@O)xF7>;5{@fLO?VVEb3q4W2r z8H|GlL<@Mt82jL!dRsGN?0k**U8JYhcwbx?jCzyubAF~Qb%O*hh$|HGieX(5NZG2J zcgC;#jdn(BbdHjAcM<{_A@=@OK zA8|ne>N$|Y`tS3SKGwP{hYaPG&z8x)5Urib0qHRAQx`p$B*@muj1bo36%-JleZ02g z->{du8r0Aw!TnA~sI{|;iZN{dZN_!)6{J)qXl&+P;5GJ1b?>{pE4K`DpL&GwAd~Pz zG4z_c3B(@kyo}{#4KJe-T)B4r*44Lj{XFB9JUQsX^n^n)OUf0H<8c^G)9d=L2u%oM z?4(mg+)dGZGEC${qj1@!)~YJM-8nRwogL;6MXS40IM|+1)tI(<*1|R2x67b(BG18u zB1DK*<6|HgSN5ZWu%yM2mf!`<6-K(H`cFtFGoin@qgPgkIHUiyGff>RxDK z@T9JMi+CGGW9S;TzZ>&B$qRlaH@0R-ZaV~RROJT!s{Ey?q^`yqcDv0@=iTb4{^V=@ zWMTl)XY8BXfbZZpQ*m8cMeWIqAURbv6AqslO6Z=@H>7b3FZ_Itsq4eDPCXd%dP)gJ z41{SE846p~f_lO5PZz1q7TUlqJOJ_F&bXOYS9|%oLK$wr-SC6tOuwYl{*MYrX1r3k zvO1Q8;sz|FiW}N3G$TS0)`V%A^}}1oi~iol%Ut8$y&EBW>Q>v&p|#>R ztfx`rG+SxzZcXrU_%@T#bz-Hf;`e}cp&IoN-o}y18W+k@q5=IocFg@nwcb}|@omTiM2UY}XF ziWb|Hg*&kzwMe8H#RZuMTW!;~4L^H$N{_0~T|=pH^mu3Z2w4tn)tg03S8Zs#OVfxz z<-`{a>$`x2itDyZaw2!qM{;le34SA+G{zUT@osXF3Qaxr-CiJdUW2IP1->ohtm= zIkF$*S4wvP2PXFpTJi7O&nvnco+L=!^F$gbW_VY#OW7Wv=m$VlT4p7wPs6SdZi5;a4wK z1>!)P){aFF_U%^bp<5~X%^lz;*tr*v;uNOA+|4}PJ4I=&?4E%IGNXU9tdrjQe9`wp zGy)ntVhBm(et&+4w(X<#CpCsgR4$ULD;~TxKTpA}lutuoWHV%@FKyB6^b%iQjs;Py zWo3CvaDh&poAb@zpQ{nFrm59AuOgjZi9%&=h z9o=<0;h>wrj7V}lwuJe!72iu(`N-V5c4gf0&#F`;#8*Prs+4+b&pgnYFFdPzJKr=d zi|)3SjPAU%LNxCZD8^0nU0$&^jHtChLP}&tLg}bi*7z zhxL?~5YH1YI_9rM)SH>F)<$yh2)(%wIp}2N7mDvg4zOqs`(@IY3?<}b5SDsd{)T>V zzJ$Q`3XiAH*KhQJHxZX!+p;ZFU0+2a(4?y`_@GQmSZ;-mr9qYIMeipBoYIL;m~8i`DwZ$;(~Nz#@+ zBDK~SSh_lD54Y8IbolPAN&wxMgw^!4Z$yVx2#x7v`Fb!E>oe}1qJ(6a1;Os2IJ!Lu zSe>7&fBJi!Ez8(z<46D3{+}w+2})vw9r7oZ(=UB|=`RpdI3{)}2&pKDAyZj5$iu2Q z*=ZCFmw*Io%z6_i`YLS!!RcY0DqR6p5f|%zg*y#e+gvYT8>a8QzL9pU0o~ zH~zrFmViBYBzD|3uMw}twmIiRRn=u-tjk=KoyhRvav%$1axqN>J*#^2#@3&$bmS@R za~O+(es^Y8)%fQf><7xsgVj-lQXE?E*|at*UO&=mgkL+OoTYmxsAs}(*XQ9LfKBuL zuh*&SsIv_(FZ0qTeadj}_Y_TDdvM}-HQY_#smbl{)E>w6dEGfa@bkn8;@2wFL+9=< zhwJN*a;Ly_$3G3nxfqNbe>VLP!wy_a`wHGBf~3g0C-$ew40LT}m%%1S&?>b|Y4}mI zSk=+>XHp@&&b)VL#A-6{FH1zLHvGuSfd_!_tnv9wAu5k0_&cV|#LRACXMX7y*aqn9T;X(uPIh`?xxeUREw~R3cLZMR71&`n22n4C*;C z^tKf9lLUXsmg893pm@VvK}gF3TQWm!d!sUO0*IFBUUx*D0`?e;WIkq2Kn;b1x@q=>&RS)j}B0JF79b; za@$En2v_zFA33RV=^@u6Ak9u~nGHmD{f-!vk!vgP5p#x`bJSIcb;?Y6){0a@Scv?> z!3Jw5-pY|sPSl$8C7|jOIzzUfpaq6+ZWIB_p9dRmQXdC1Fs!~49UmuM0{NaM3pMw~ zgDqp3@I*94$_`~msX%j*>!^O41hi!%tpC!Cp}AsLBx^?~W1IFwoKK!2+`ZVbL;SY6 zd#x5csi`W-T-)}(8W-7K!nX+yJ5W5-F7)^(W6i4>u~_GptG<8DTCkE+E^cgWIq-^+B6Q=~ z4A}I(!Hb(Z3lG_FtKepBAXwcamFI(Otz_Z-n5!ac1Rg6Nn-Ei`nDt&J3iMX$2m~i_ z``&o&^Wnla-oD%6!w|+aL?)3;X%MymgZ{fL!W4p9(mv@5yBuzy*rFS@KO8*B9Va$%W`_Q@fo zfVx$4JJh7{Hpki7rF5TG-_MIuMbileo>DYai$;*eRGre!Z{F6;g(aR|Re~1FMd9hsa#`JGg+S0MF&|I9z44`z*L9&WpqR%4GoG4bqdhkD8BVD zLL_ryae9#uvNW)PoZ^6g7&T2QJwrk_WGRz|f@6U2t7gkoK??Q&*z2%2YD{~^vePWj zIcfj^9R29#H0;9K6HE+UkLsQ8`#VK`Vl^wX@ zCHIaXT9iU+6$JMyq(xsI-{u|@b#1BFIQW9TBD?9jB#TuJP#-WDL3a4=dY~nUf!-;jYizjqBe;q z1NjCHMgx9w`$<2Y**0W}ZjVJB>j?evo3U>kU0$zaFQm_4ful#3)^s%^C zqeh!+{bNKynbI^OH73A6>(xR;mXH{pkO&vMOMSahcRVGZm#Dxin=o&In)c_L=0G@w ztc8b-V8y3pykzaoMXj-kX8&GLZGCIlY6KQR0!lMc{8MbLf>9axkSMFd=Ncn0|y)@@9&xvwQ0WgY_U3+X#hf4s16fVBdyj=uxr_`>*~ zE2Q`@wrRLo;e{8y?`G4|Rr{Ks-0x0=EQZG4htzk?8uU&W70XWC zF5g%-;g=ELlHT$1Mcd5%xHx!sVbmi#2X?v#YEp&las(H-Br1D@~Q-!MSDNdqCP0|4kUvt z@*fdMCf;tlbsw;9#Hn7uH!(y@e)5d*VYT1@jFKK>Pkd_qxrKk{^Zg*qY|dZ`pYQPI zA3U9YiVMwGT&^H}+b%a@IVWk$HDh~2F-2FJ9UHP{G)2}BMDkHMFSS381-xeVJ z<$^mH(m|Fe8`UWHUAp7mJmwlxo3_r*t|RsKhWMMS&tTlQvvX2icyOFa9r*0@(lYGi z2ki9zh<|fDY-29oRt`CI0bG9JcN%2$b^%JQ^oQftYJ>7%uC_rBk-+6Zl@XVuBUM`6Cn~*-De6a?7ib<=yYQ-D8)d! zJLc|erv}ib8V6Qh4Sra`(yafoC7k5Ps)6Cd+~GYoJ;B@!;e{sMHUWNL&f`?c)fo<3 zrWJ3xKQ@apqF=|mJtmCFI)%4bGZmI_3R7O@45 z^d<_sJ3k}KnJ9KJtZ#&I0$wDXs2dqEMA!zVJX!i9PgHl-mlY2y=0$mrSkoPg>~e-j zkGM>fYNPUG4W3O%hLhS&1G$D&Lp2@QDJl4CXx zI_%mMa9`Meso9qGG^I7UoS0P~h;v@~+|+vn>`PKJ_vpSaS!uONcA%d}d~7=C;L)GW zHB$v9K$?e*3NmEKtR~!KNq{3n8eHW;#Sp#J+q`FHw!Lj z_H6Y&DNy{~XOsaRBMrt|2OmFjTSb{sP-M{aIbcUL#fYilC^cgGA(t#nPQ*ZmBaIKH z6~>Mq+p*HvBMZnJiEC5G_J&l5S%-87sr(#$eq}NmMOQmvDcw~R-sK`-IP&yfx}|&s zy1oyV;3I&kwu|g8f?VwNq1)ZFJOw37h|Yxe;h0EIcs))oE-Q~xP{*n&<%5p9`Uv{s zqxA%9`|G*+0~Umt!y*sy&2>s)V%Qxw#Ne`IGuL%wn>ddx9HXV%zDh6?y(by<7-UJt zA>O|^&}Er*S>u&sLgPe#n!Z@|`9%?S{M9Thitl=vuju>W-N2HK3b04YzT1OL8tenw zS_&jl4O;4SN8hHB+chasYyQCUwiV)~sDp*XWzzK+fO7!N%(>4*pUpYgE$oH~VVYvp zVxI6p<&-KkmdQq7Q)?~iD=j=j1Hbc_=Z@hX*klrOlXK6x``iLBDCyl>p4)+4a6&|T z{SiWafD{4#uvjw@hD`+H;EINTyD5D^6fgtL(J`gMa@wB`Y%A0+C5JqpXYGUu!7usn zmMs}67D=6Wudcf2m0L?0*gySS<=k<yAPWxbJN>9K|2+KIR6t1n9Bt0#P4)+wobv)orkU4Q}s} zr1zqytnt@YRyj=X)X!3TPJ&79qJhw7x23JX$33_ujL0FyOu7zn^Q{Lg+_WrsD@>mU z2(Q;Ap#0ob+_7J=EAqgE`{4L^LduL6WPu2GFktlvutR`4E8h8}DG?|%WwtIPq-THSmDphFLis9A{2+5L**2AUoel@>A*0 z_5iK>(d8fiD(LKXSL?ieC!9laT z9MZ*n`fFkG42yhEihK`=G+TeB53uy=$z+yb+aqmys5yJyG#@{9L8fNrY>D5pb z)kF`ig|lySpC|5A^pVAWF@yJSLEE|q*7ecVZviJNk{{#ZrX28aJJ~v5EVhRU+xc6>cML;{} zb`UD}DPdIOzi2eDBCJTaJh$dKUMAMYQE#i!$NPeQ=wyRk;QDXO~O>X@%C4}bVC zG=MW8LV)ghX5YpLhBg_zi0@l9sLxs}BIspg>xf3%Er%psfIf&83t!e!t#)BFSsIj1FA5W)T> zR`(EW{NNS9i3|Mt1bw|n5q(SrwS?qOqu~tuM_aq>Hq$;i<#rm_G&8FjIZ5S+9Zm}4 z-Si5`x^OGlRE#qkP>_-xE$c6q4VbpXwmOIgG`|%eLLaue+|DNi)V&!TA#i35yd=Ya zKkj8krw6DDTNsKW<~$##beMo3&(4O{hb;xjZ{Tj>YxweNv#E5F=n`l2f4CQ_X!stv zDJ+0l(BhNg7XZm7Yb+m zpRv|$KhkZh9k;-Ii!)k~Y;-woXZ|>!>9Xn(2w-fIIRMz;0H`!bB$8@0QZ&^(rd+>t zrx92usr#T*3p_j}=r3t~3h-a*c#f0(1yM4JXUbL<@d|jQzam?6a0tl9K^N&UkRQ)^ ze&9aSo@-R7d8oSSs2Io?PZ?5~R+(~5yocXIeg<3{v(xR9!loOhE2cZ9=^i{F^^Y}M zs`E6oCIBb&6OR&~lw@+pd^3L7zK~zpul0`sHwfmWvYw-#Bcm}=9dJGwK3P8TJ~2OW zE17=T;4olEHGxxPQ)Zl2?O)BNP0>%vFAR(7C)RCjKwarynO>1!vp(`V=9Z_%PTD4E zuTXBww)Jbss5N|b9Wd@(cE5N^KBa5a!&W3+1a}m4BX=T$BKuH|BAFwnBC{f&p+&=M1PNUZ8J-7B;+ zDRZcOsFs(vF~klz-641V#UJp#H?9w^KWxV8g@9v`Eo;(!5 z=R{ihDyv^+%I9z+o^=B=aNDt;_C#+`lXMJ?xKE7uA7XR z{opdtp=|^t0^<|UxJi1O2R04jHw@QL3MPy`?t!J5@aXIU)ulIhU*k%RmCLY2{4ch|>*;<%Msr>i zvY&y5l5~5N*|JMUE|UScm&_NaT$!OET~Q;+-lNrvWT%IVs3EksN}RJ|Bt|R@A@BQ2 z`%F}XXoH??y?$3|0v6j5!U`MKFP5;}2F#rIYBG9yN*uoaE^cCIJtGW&xNI^lt66{% zky`6V9&kmg@st@qET(SsaT2T5gJmocivBeW!Es-^f7Z+}mkrsj7nJY$%;PKS{!j&^C+%CLAH2} zQJ?C^yy{-je;Hy_O7E{3_xfE9Vm9|Z^jR|ZkWjrkAOyZsKTUv0_~%yu%n~U9X_(9? za;SkyBzVfZnsa8}ttqGcvLT@))w9M{^e6Ah16G_%uK_4gB^l^83NicHU8SbKcqW|` ziUUKi`AT1;`|dC+uF9WcOf~}uLf!v+05faU#3L~-Ya&)N{15ghca9!;Rd8+PBWv8c zLL7J|sY8o);t*X7X_kOp4gs=Ih|`tg+cJwe!;;0Gl2X-7r4FFX75a!~E)3(0x1R6vju)5=whkeC6lwvvKFZvYFj>CZ=b#D6yCNr-#fTmyhNMi}HsUBt!D`5Dw zVmsAY3e#<9)3(w5oB~}{Em_sJ%+08~Hh)_Pv!*y%&)#hqmO(SyxPf0c9(!eRPMWN6TIJ+UjOiqEXakTlW5;Ku6U5PP>kr|hXO|aN4>>IWk z$<*iZRgz}*O&|V`c+J?x^uLH{R~^AUPk4T)Hk-eIh?z z@^7WHkLrTR=uRg8Y$5Gk6m^)$o`o+*1kqfXi2<(rvi*O<@Y}`X$l43{Gy`^JwnAGT zxWX!J!vC#Gd_@}U8ywClr!_BV1f*M>2&@Wd`vi%V&n+E}v@Ef17%mv!wcm^CU-j&J zb@4i1Ymd>QsN8{~2*Mb8IZX+T8IxIClnnG z6m+WU0w`(tMZ-UPLG=-rZ0|iFLHlpgo$XNgaH&?4&yOLYe*U%-=ML%sD|Uh^CFQZd zpTUbueeR@t7mG3*>YYJG-d_%H9BUMjZX>ww3vBRh7|pTPcX@M8A{vqbvPi`-N))`! z_g^=jrl49_3Qs+VMbvO&c91yy@H%&q3V}ijXs+x8qNU~q*)5*?{qjyJ`$l+SIt9fK zOFG+(APK{)ZWr4}ZO5_DBDK~H3DT@5EtX8&h%~7M0VPdw_B`utyaaZ^7WuX3VLDr| zf`>h}n6rrADaB}4CA3&dpw5V|#tuY8QOw8`hP1y*qexqT?5=5MXyq~>gnrsCPpF(>`$D_Pghgr44<<%mZTAX8%3!~E#;cHmY-Q~Pv>Cbor66QSbeq8J=of9Qa%&u?F@ZY*yx8{yR zZ^h8mD+T0qI}G)y*NxW6Th7&YU9%BBS4W0?lm(r97M*GL=g`uQ zVM1*&B1OI{Pz`xVw1DC$)a7GugeMim4|0E2*`Pyh$IGaq^2f%&>J?1Mfm}Lf`(5K{Lki&#|yT7}?mmX7LvM>mQ z<+0<)odb0&!FnBv;3*BjHSvRsFQpZ-612HYYSCN-u848&uu>4I>lR#Gy;ON+qsHA> zn9)e*SWTMM2X27;hF{fqy9g?cBB^R#bu8JluX9Nu3yt|69xM{Ut8wa1152>%iu_xM zl@_wSl_rvm;y{??Z_L1?cyomC(@cBp(_pyAd?RiS%eEV(0a}Ru%w&|D~(!v1h)Y1T*+>zOO$HI z8P*pMf~wskVKlL*{i$*PI5^XH>>l9LXv6!QZ+snX<1vqvZn zBJVtJetZI~Y(Z;(^~XTbG>>+wrD_(kwTqVv3II8u?8Pb@Kl>VtbScBcFdbl%FY}8O z!&i$A^Ol?e|g=4uM!ynjHJGVo8hvaDfQ^AzX3jKZ}Q-8*q$AO_M z@b017z7Mx=?)rMa0G;=ISmHP7jw;e@dX-O|=Q2}P^Bj&7Uq9_Sg_ZZ6 z7g+vED)DRb!irm0|M{qK0tN>=Grzm=?+h?<{TUDRxP4weB!kCjby#<=;Tt?!wrct@ zjuvnl6-3F%#BSa|g>?l^7^ROP0|DvMV8d*99R!9X&Pa$_9;f+B`@5<_l9UcH!ogsK z+ym80QjC1ekOpV9Xar+Y$`+Xzc802I~@oAw{_u+N!W1LPSIg) z7W2;cfbsF{mBg1B-yz(G_1VoKm&7+W!`E-vusRoBk-Mv3!7xJ0u3(G=sT`>}7e2wW zFGk(G8@fKtzh29oqN5WdNsZ)qz!>WghX0{-H} zp+|qYWj9YEh7@(NSvY`#DWSjufw(7H)N%CFip=yVdybt`t_CvLyS?jOf5V{`e9X zr3nZ4Mm)84RIK@FAhNM~;o*FQFu!}9<1Z8L6syECY~zP3!E3B@ZhMB(Gx<-zqM1q~ ze+h#i3(>mvBeAOw=y^e^iW9JJ5odLFq<J^Fx-`tapF@Bvc-ig}al z!|_WdL~9c11TnB9zxT3PBlkanyd_z*9mYi)mP!?^hn^gEl(AWAim&q;N%%DNJ=IKw zaqV&yhUn}xB~(9d@zfd2DI9YWPk^t@bVQCK+VgD z{W%fQYP$JJUc{;vU@-ki{mDvA(UJ~PBl%D%5B=T8W|{AkPJ)Z={_GdBT{*XqZp%q&*9Y%8?`--4gN(Qgw9;$)%w$$0?Sqd zme>_0S<%7WJR>-*f}bibCdOWYPdG@;UMn&<)M2(28Nuj6B-i;=*mSBI$b=X3CDj}F zljkvGN|T3KU&}a2F||8Emrrp-=cguNshBYsp(Mm`8YB{o44l#c z^sOD}Ga*^qkTgx$Hy)H^6$4L7S%xs@??2Xn`NlFFiHOW*V0GVb=(94G1HsHkT{?+U z?%$)ud*P_&C&%Mh#6A6bNcqewi~txu#Hs8uSP~GfPZAH{q#yc7Yk82z+EY9)xf2hx z062+zpNLQd%eiO9%>w^pbZj&v;3nnWCQU>pDl3jyR>~UOmWBmyt zSi(fz;n@zsK%6mX7;YP7rkwe*PVHFvOV?(_-ZZ?8+rrwM#|f>ovwjywJ{q8_r~&mU z(GpXfM?KaFC7;OFvQe6CL#vwt0S;zWr~zccAu3bJ{k-q8%5_6sByo&iX@0W$1DoXG zE&FR~UE;+%&fhkB=%S)?Nx)W37gpU{5KU<#5y(J;(Tqfh3#KS%$OCt@kXOJwymQ}j z?O&R>Vw2KLBJ442Rm99X)m{?Hv1p@)$%PHMG0?m?o2w#Bquz(%h1PKZQp^cNx?9N# zrphZW6mR*aKp4RMHy;uij;T0K7f77rh{!24OY6k;w6y1z_lpiK=hk%%avrj%1{@IT z4~0v8&FCCCxDVN+(lL+qj-dtyv=Dd?L7j?7q(-QPqSLq4s5I&X+29=;P^OZVU2qH? zrb`fbx%gN76q)3z5tB&GfpX9q{P5-SB^hbk3JG$LqLZ&)Di`j78{A3qO#2pFC1Cu& z0#gpG@%yL|gdP(59fwEvB|1N&Bqd|rE;FI3OtJVE#7@Mu@3~~;=j;q~RxU3#>%>cA z^PDk%l>`U+B28kYzP>VVP9>aJNL&`TaIp=4F0%evypY8u?yO%~>p~dj;sz(~Z1(Ob z(<=zVnVD0VOe`*^^|(Qel0*0@n&+Y0-N zC9xuV`snP6wb?k|f7I1Jfb znB$-)V`$Q9851DT{Ixxcfkp)~OhTL@Bkw%zl+c*O>MCxY>CtqSD8qGfv{cw?wfYm` zRd+{Qs~m3=xJ)GJkWQzj=P+9-nStFs-P*BpmnFs`oZqs5WE~`bbEoUQKumDEjj*%b z1qAcCxq32R@MMiZdewTo6=VTmsLuxe6cfU!?CWeEC!sJY>y9LI?Aj#jXASj@RxKhN zgM2VKN5SJ)>vJQMWzs0LcVJ+3x)lW+T@VUIM)hN4XodnV1q@B#nsBP9DzLR-YofJC z)0vnqja`__u0DIAr8ng#&Bn?`COwsErw)R&!}$tm=|m666D3ZZ|dw8yIfCmFE?=Ml7i*UcD5F zhGWUpl0Fnl87CERc#6lK!K5V=0^ToW5{<6uX$-InkcxZ2?R_b3N0ky`({I!8DlnSo z=dlXWXZ#L&$PQVA-xna5AMp7JiY6Z=Bu$Z_5J~=KMuKPijpk;Qhz`AgJ0VyiZE=e3!xMxhdb$Jt*l_1t#&U}_GJz- z2wN#U2``dVM)y5u8V1uem!Qn+#R)oI>m<>HQnpsvPvG86yHTx_sW9EBSiIV3x0YY${$k>#@g{&@G&9mF7g+keal~GHbHD5#?I7@1V(vfP4EKO%>(_&2 zy)Gm{TX#Vb$U1GtRLT=wpL*zTL33(6$Zzb7PLhm^D>~f-PZ~|guJo?a zd^clt)`^MB$&Au3=!kA%^!etEC8UxZ9Kd3veUCH0` z|BdpU={YoV6Jz`v4dJ}V2xltAFIE}2W3#fEhHRP4ra;Dp8UPgu~={(baT z|7VpwDv%r#vwS9rU+^3A-L4OgrB@km747m#?kxI40l99~Ke@B%2PzH7dS2J_O9}h{ zU7D-cODSBAZB316C(<)#bM(ExPG(pOxPv!Pey*FB0zQl(Z_YUi#utRY0(Y~td4~ze zzsH<1J;Q3bz{~_3N<270du$wWQuWtii2=KYtCF8>)-P8G7j ztNfXLtHKcqTa6~`AFOq3;X(@-W)C~qG0|P#<4)pWHTVI1RjTC_9CzrY3kK90GDVoC zt8!*Z9=1V!rcxZg+Pr9Fu!z&-F5WWe0zr5Ify8B+kdEtf;1v+ftf7*3iF#6gMUU}*P_;V`gC&L&hnTbWJ5a;wi_OWFQ z;NfTX45mdqTB_jVDqW>R4+h!7x!|SSa>kXzom8MHMou#gh-tp+VBr7OeynUil8c{7NN{A$jQ3(t( zI=&Th0a_K|NAwbQ5Qt~MEjswuW}11{Xo}q3EzYcA2FJ05l1Ru>M{#?-tUci*`K&i5 zF8}W&C`WL`LZGtDzxpvIZw)AMnHVcO+eZx!IFbi>P&BPh&f{0?ar~vC6YU>{+%|r> z!%5P?M6J5hFZPMW|`!0<45qzM)IOFe=^EI)Y?|&KH{SCnPKc~+F@cYFnnf@ssyHfW!Knw#w z@Xb8T;Fy)*;G=`jkbDU%X|EIQFxWESemj`$O|*OQ&=1quze>0jLF1LX>Z(#6NGo857+=f)5%l+6li>!#k$WpFBG18z$U%%p=?f}a>WIF=9%;N~kIYC#U8o(11 z(HqYb7_$r9B;j0TIHLsH119){Q({*D{!re+R|^nq3g~I@RO7>JPuAiK`H1F+ZAL}5 zR%au-dPgeCl(<42)FEx2Z}`SPB{a}kDTJFhjMRLhE~pWPt;6ty#=grbRSs1(;ii(> z2f&+-rx=kPiV;u)%Tp^-SOUYThxnxf72~Uhu-!`rn<@R2NVo*}5DC4U@S6xsfn$NW zAIdv;>;_y!JAtVRPx6WKIDiicq^}PGO0pdm`(qBye~B$*4-=?_ndW z$HR_E8FB_<8j9EsF)8Pl4s_3 z@%S(+2@JD60^ZW-V+(xZab^}#@y@bpvds=V+1NQa?Xue*dr_f(OQ>7k%wwPZS{(F* zPkkoIMajJ1EGl!zVZHhz(q+ixbKVSj+5+0%Uj1UP$rse2z&V9tSk!lZ1A4ckdMvdp z431S6rs&^f>chKSr4?3LZKc<(@wqCORI5?z0d?xF)u-P&4I*Zm<$@o~_Oe$=#TT zhT?Fp^&aw|hYgzRj3GZ^YaI6(4uybauJ9d_p3YO=GGBIFpqePR66H~nlI2y3QkAB3 zW$>t6j~MZN#{Keb zr`j9LQun2)Te(#_+|atlWay`Bif!0Dva7I7MK208=b~VzVVaJ-r7uI1VM$V?qhdND z8Mz_>y#>8f54~LiO`9_}4c1F1mPdIX61YvdmhId_+yWush8N6?{VzIdGJwn$oLcb+uUL82O)0ssI2@>%qy literal 0 HcmV?d00001 diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx index 47abeff..20fa138 100644 --- a/apps/web/app/layout.tsx +++ b/apps/web/app/layout.tsx @@ -1,9 +1,10 @@ import type { Metadata, Viewport } from "next"; -import { Bricolage_Grotesque, Figtree } from "next/font/google"; +import localFont from "next/font/local"; import "./globals.css"; -const display = Bricolage_Grotesque({ variable: "--font-bricolage", subsets: ["latin"], weight: ["700"] }); -const sans = Figtree({ variable: "--font-figtree", subsets: ["latin"] }); +// Keep production builds independent of Google Fonts availability. +const display = localFont({ src: "./fonts/bricolage-grotesque-latin-700.woff2", variable: "--font-bricolage", weight: "700", display: "swap" }); +const sans = localFont({ src: "./fonts/figtree-latin-variable.woff2", variable: "--font-figtree", weight: "300 900", display: "swap" }); export const metadata: Metadata = { title: "Quack", diff --git a/apps/web/lib/persist.ts b/apps/web/lib/persist.ts index c7d4a44..cd9bdda 100644 --- a/apps/web/lib/persist.ts +++ b/apps/web/lib/persist.ts @@ -21,18 +21,47 @@ function db(): UseStore { return store; } +const object = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); +const finite = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value); +const bounds = (value: unknown): value is Bounds => object(value) && + finite(value.minX) && finite(value.minY) && finite(value.maxX) && finite(value.maxY) && + value.minX <= value.maxX && value.minY <= value.maxY; + +// IndexedDB is an input boundary too: old app versions, failed imports and extensions can leave +// malformed values. A failed validation must preserve the stored value for recovery. +export function isSavedNotebook(value: unknown): value is SavedNotebook { + if (!object(value) || !Number.isSafeInteger(value.version) || (value.version as number) < 0 || !Array.isArray(value.strokes)) return false; + const ids = new Set(); + for (const stroke of value.strokes) { + if (!object(stroke) || typeof stroke.id !== "string" || !stroke.id || ids.has(stroke.id) || + typeof stroke.color !== "string" || !finite(stroke.width) || stroke.width < 0 || !finite(stroke.createdAt) || + !Array.isArray(stroke.points) || !stroke.points.length) return false; + ids.add(stroke.id); + for (const point of stroke.points) { + if (!object(point) || !finite(point.x) || !finite(point.y) || !finite(point.timestamp) || + (point.pressure !== undefined && (!finite(point.pressure) || point.pressure < 0 || point.pressure > 1))) return false; + } + } + if (value.dirty !== undefined && (!Array.isArray(value.dirty) || !value.dirty.every(bounds))) return false; + if (value.seen !== undefined && (!object(value.seen) || !Object.values(value.seen).every((item) => + object(item) && typeof item.dismissed === "boolean" && (item.area === null || bounds(item.area))))) return false; + return true; +} + // Failing to read is not the same as having nothing saved. Treating it as empty would let the // next stroke overwrite a student's notes. export async function loadNotebook(key: string = KEY): Promise { try { - const saved = await get(key, db()); - return saved ? { status: "found", saved } : { status: "empty" }; + const saved = await get(key, db()); + if (saved === undefined) return { status: "empty" }; + return isSavedNotebook(saved) ? { status: "found", saved } : { status: "failed" }; } catch { return { status: "failed" }; } } export async function saveNotebook(saved: SavedNotebook): Promise { + if (!isSavedNotebook(saved)) return false; try { await set(KEY, saved, db()); return true; @@ -45,12 +74,13 @@ export async function saveNotebook(saved: SavedNotebook): Promise { // other tab's work. So the write only goes ahead if the save is still the version we last knew, // checked in one transaction so that nothing can slip in between. `known` is null if there was none. export async function writeNotebook(saved: SavedNotebook, known: number | null, key: string = KEY): Promise { + if (!isSavedNotebook(saved) || (known !== null && (!Number.isSafeInteger(known) || known < 0 || saved.version < known))) return "failed"; let conflict = false; try { await update( key, (current) => { - conflict = current !== undefined && current.version !== known; + conflict = current === undefined ? known !== null : !isSavedNotebook(current) || current.version !== known; return conflict ? current! : saved; }, db(), diff --git a/apps/web/lib/scheduler.ts b/apps/web/lib/scheduler.ts index 6e076e2..5335e25 100644 --- a/apps/web/lib/scheduler.ts +++ b/apps/web/lib/scheduler.ts @@ -127,6 +127,7 @@ export class AnalysisScheduler { } checkNow() { + if (this.disposed) return; // Pressing again while a check runs can't make it faster, and used to start the same // expensive reading over and over on the server. const newest = this.newest; @@ -174,6 +175,7 @@ export class AnalysisScheduler { private mayStart(): boolean { const newest = this.newest; if (!newest) return true; + if (this.flights.some((flight) => flight.controller.signal.aborted)) return false; if (!this.deps.overlap) return false; if (this.flights.length >= MAX_IN_FLIGHT || newest.trigger !== "auto" || this.manualWaiting) return false; return this.deps.getVersion() > newest.capture.version; @@ -203,7 +205,7 @@ export class AnalysisScheduler { private cancelInFlight() { const flights = this.flights; - this.flights = []; + this.manualWaiting = false; for (const flight of flights) { flight.controller.abort(); flight.penUp?.(); @@ -239,6 +241,7 @@ export class AnalysisScheduler { } private async run(trigger: Trigger) { + if (this.disposed) return; const capture = this.deps.capture(trigger); if (!capture) { mark("check_skipped", undefined, { trigger, reason: "nothing_to_check" }); @@ -268,8 +271,14 @@ export class AnalysisScheduler { let answeredFor: Trigger | null = null; // Whether a newer check started while this one was under way. Its reply, or the newer one's, has the last word. const overtaken = () => this.flights.indexOf(flight) < this.flights.length - 1 || capture.version < this.delivered; + let onAbort: () => void = () => {}; try { - const result = await this.deps.send(capture, trigger, controller.signal); + const aborted = new Promise((_resolve, reject) => { + onAbort = () => reject(new DOMException("aborted", "AbortError")); + controller.signal.addEventListener("abort", onAbort, { once: true }); + }); + const result = await Promise.race([this.deps.send(capture, trigger, controller.signal), aborted]); + if (controller.signal.aborted) throw new DOMException("aborted", "AbortError"); // The stroke under way isn't in the notebook yet, and could be this mistake's correction. // Pen-up comes just before the stroke is added, and this carries on after that. if (this.drawing) await new Promise((resolve) => (flight.penUp = resolve)); @@ -331,6 +340,7 @@ export class AnalysisScheduler { if (this.flights.length > 1 && (timedOut || !controller.signal.aborted)) this.deps.onBusyChange(true); } finally { clearTimeout(watchdog); + controller.signal.removeEventListener("abort", onAbort); const index = this.flights.indexOf(flight); if (index >= 0) this.flights.splice(index, 1); // Busy until the last check under way has finished, and said once. After a cancel, the list is already empty. diff --git a/apps/web/lib/types.ts b/apps/web/lib/types.ts index dcf9aef..7b12590 100644 --- a/apps/web/lib/types.ts +++ b/apps/web/lib/types.ts @@ -79,6 +79,7 @@ export type LineTrace = { }; export type AnalyzeResponse = { + // The server validates that alerts have new verified mistakes, unique ids, and no resolved overlap. snapshotVersion: number; requestId: string | null; status: AnalysisStatus; diff --git a/apps/web/package-lock.json b/apps/web/package-lock.json index 8645f74..47668ed 100644 --- a/apps/web/package-lock.json +++ b/apps/web/package-lock.json @@ -1252,6 +1252,13 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.1.0.tgz", + "integrity": "sha512-GPm/Dbvk5RLmhRnxXIa7nTpK0xtl4E0wj0MslsuNiJ1FIROU+2+9lgUC/dgF7Ey4QCWhfEsgxilMRMCdHPfDfw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -2590,16 +2597,17 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.5.tgz", + "integrity": "sha512-snPlVppFcYWfLLFQGhDxUyBcHNtMlWmn3jtj7pEaO+DP8pqL0IvAzXhMYmn1z9ASNtgmwLBdFxSf4tMUB0OTKA==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.8" + "@isaacs/nonrecursive-glob-match": "^1.1.0", + "brace-expansion": "^5.0.12" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" diff --git a/apps/web/package.json b/apps/web/package.json index cf63068..e13118d 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -6,10 +6,11 @@ "dev": "next dev", "build": "next build", "start": "next start", - "lint": "eslint", + "lint": "eslint --max-warnings=0", "test": "vitest run", "test:e2e": "playwright test", - "test:production": "playwright test --config playwright.production.config.ts" + "test:production": "playwright test --config playwright.production.config.ts", + "typecheck": "tsc --noEmit" }, "dependencies": { "@supabase/supabase-js": "2.118.0", @@ -33,5 +34,10 @@ "tailwindcss": "^4", "typescript": "^5", "vitest": "^5.0.5" + }, + "overrides": { + "@typescript-eslint/typescript-estree": { + "minimatch": "11.0.5" + } } } diff --git a/apps/web/playwright.config.ts b/apps/web/playwright.config.ts index dae225f..e6ba8b9 100644 --- a/apps/web/playwright.config.ts +++ b/apps/web/playwright.config.ts @@ -14,9 +14,10 @@ export default defineConfig({ expect: { timeout: 10_000 }, workers: 1, // the fake Meta server holds one script at a time fullyParallel: false, + forbidOnly: true, retries: 0, reporter: [["list"], ["json", { outputFile: "test-results/results.json" }]], - use: { ...devices["Desktop Chrome"], viewport: { width: 1280, height: 800 } }, + use: { ...devices["Desktop Chrome"], viewport: { width: 1280, height: 800 }, trace: "retain-on-failure", screenshot: "only-on-failure" }, projects: [ // The notebook on its own, as it was before accounts, which is what proves that path hasn't moved. { name: "notebook", testIgnore: /(accounts|learning|lessons)\.spec/, use: { baseURL: `http://127.0.0.1:${port(3100)}` } }, diff --git a/apps/web/playwright.production.config.ts b/apps/web/playwright.production.config.ts index 6c52e87..ebdf3a1 100644 --- a/apps/web/playwright.production.config.ts +++ b/apps/web/playwright.production.config.ts @@ -7,9 +7,11 @@ const port = 3200 + Number(process.env.QUACK_E2E_PORT_SHIFT ?? 0); export default defineConfig({ testDir: "tests/production", workers: 1, + forbidOnly: true, retries: 0, - reporter: "list", - use: { ...devices["Desktop Chrome"], baseURL: `http://127.0.0.1:${port}` }, + outputDir: "test-results/production", + reporter: [["list"], ["json", { outputFile: "test-results/production-results.json" }]], + use: { ...devices["Desktop Chrome"], baseURL: `http://127.0.0.1:${port}`, trace: "retain-on-failure", screenshot: "only-on-failure" }, webServer: { command: `npx next start --port ${port} --hostname 127.0.0.1`, url: `http://127.0.0.1:${port}`, diff --git a/apps/web/tests/e2e/accounts.spec.ts b/apps/web/tests/e2e/accounts.spec.ts index 28a8372..457d0aa 100644 --- a/apps/web/tests/e2e/accounts.spec.ts +++ b/apps/web/tests/e2e/accounts.spec.ts @@ -212,6 +212,7 @@ test.describe("calibration", () => { await page.getByTestId("calibration-start").click(); await expect(page.getByTestId("calibration-next")).toBeDisabled(); await write(page, "1"); + await expect(page.getByTestId("calibration-next")).toBeEnabled(); await page.getByTestId("calibration-next").click(); await expect(page.getByTestId("calibration-problem")).toContainText("unfinished"); await expect(page.getByTestId("calibration-progress")).toHaveAttribute("aria-valuenow", "0"); diff --git a/apps/web/tests/production/safety.spec.ts b/apps/web/tests/production/safety.spec.ts new file mode 100644 index 0000000..62cc6a0 --- /dev/null +++ b/apps/web/tests/production/safety.spec.ts @@ -0,0 +1,114 @@ +import { expect, test } from "@playwright/test"; +import { strokeCount, write } from "../e2e/helpers"; + +test.beforeEach(async ({ page }) => { + await page.addInitScript(() => { + localStorage.setItem("quack.privacy-seen", "1"); + localStorage.setItem("quack.monitoring", "false"); + }); + // No backend or provider credentials: the compiled notebook handles unavailable checks. + await page.route("**/api/**", async (route) => { + const path = new URL(route.request().url()).pathname; + if (path === "/api/auth/config") return route.fulfill({ json: { mode: "off" } }); + if (path === "/api/health") return route.fulfill({ json: { status: "ok", modelConfigured: true, environment: "production" } }); + return route.fulfill({ status: 503, json: { detail: "unavailable" } }); + }); +}); + +test("production sends browser security headers", async ({ request }) => { + const response = await request.get("/"); + expect(response.ok()).toBe(true); + expect(response.headers()["x-content-type-options"]).toBe("nosniff"); + expect(response.headers()["x-frame-options"]).toBe("DENY"); + expect(response.headers()["referrer-policy"]).toBe("strict-origin-when-cross-origin"); + expect(response.headers()["permissions-policy"]).toContain("microphone=(self)"); + expect(response.headers()["x-powered-by"]).toBeUndefined(); +}); + +for (const viewport of [ + { width: 1280, height: 800 }, { width: 820, height: 1180 }, { width: 390, height: 844 }, +]) { + test(`compiled app loads without runtime errors at ${viewport.width}px`, async ({ page }) => { + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + await page.setViewportSize(viewport); + await page.goto("/"); + await expect(page.getByTestId("ink")).toBeVisible(); + await expect(page.getByTestId("tool-pen")).toBeVisible(); + expect(await page.evaluate(() => document.documentElement.scrollWidth > innerWidth)).toBe(false); + expect(errors).toEqual([]); + }); +} + +test("loads both fonts from the app's own assets", async ({ page }) => { + const requests: string[] = []; + page.on("request", (request) => { + if (request.resourceType() === "font") requests.push(request.url()); + }); + await page.goto("/"); + await page.evaluate(async () => { await document.fonts.ready; }); + expect(requests.length).toBeGreaterThanOrEqual(2); + expect(requests.every((url) => new URL(url).origin === new URL(page.url()).origin)).toBe(true); + const loaded = await page.evaluate(() => { + const style = getComputedStyle(document.documentElement); + return ["--font-bricolage", "--font-figtree"].map((name) => { + const family = style.getPropertyValue(name).trim().split(",")[0]; + return Boolean(family) && document.fonts.check(`700 16px ${family}`); + }); + }); + expect(loaded).toEqual([true, true]); +}); + +test("a failed check keeps ink saved across reload", async ({ page }) => { + await page.goto("/"); + await expect(page.getByTestId("ink")).toBeVisible(); + await write(page, "2*3=5"); + const count = await strokeCount(page); + expect(count).toBeGreaterThan(0); + await page.getByTestId("check").click(); + await expect(page.getByText("Quack can't reach its server. Your notes are safe.")).toBeVisible(); + // Wait for the actual transaction, rather than assuming the debounce has fired. + await expect.poll(() => page.evaluate(() => new Promise((resolve, reject) => { + const request = indexedDB.open("quack"); + request.onerror = () => reject(request.error); + request.onsuccess = () => { + const db = request.result; + const read = db.transaction("notes").objectStore("notes").get("notebook"); + read.onsuccess = () => { resolve(read.result?.strokes?.length ?? 0); db.close(); }; + read.onerror = () => { reject(read.error); db.close(); }; + }; + }))).toBe(count); + await page.reload(); + await expect.poll(() => strokeCount(page)).toBe(count); +}); + +test("corrupted saved notes produce a visible recovery message and remain intact", async ({ page }) => { + await page.goto("/"); + await expect(page.getByTestId("ink")).toBeVisible(); + await page.evaluate(() => new Promise((resolve, reject) => { + const request = indexedDB.open("quack"); + request.onsuccess = () => { + const db = request.result; + const tx = db.transaction("notes", "readwrite"); + tx.objectStore("notes").put({ version: 1, strokes: "recoverable-original" }, "notebook"); + tx.oncomplete = () => { db.close(); resolve(); }; + tx.onerror = () => { db.close(); reject(tx.error); }; + }; + request.onerror = () => reject(request.error); + })); + await page.reload(); + await expect(page.getByText("Quack couldn't open your saved notes, so new writing won't be saved.")).toBeVisible(); + await write(page, "2x"); + await page.waitForTimeout(700); // past the normal save debounce + const stored = await page.evaluate(() => new Promise((resolve, reject) => { + const request = indexedDB.open("quack"); + request.onsuccess = () => { + const db = request.result; + const read = db.transaction("notes").objectStore("notes").get("notebook"); + read.onsuccess = () => { db.close(); resolve(read.result); }; + read.onerror = () => { db.close(); reject(read.error); }; + }; + request.onerror = () => reject(request.error); + })); + expect(stored).toEqual({ version: 1, strokes: "recoverable-original" }); +}); diff --git a/apps/web/tests/unit/release-safety.test.ts b/apps/web/tests/unit/release-safety.test.ts new file mode 100644 index 0000000..a4f2268 --- /dev/null +++ b/apps/web/tests/unit/release-safety.test.ts @@ -0,0 +1,228 @@ +import "fake-indexeddb/auto"; +import { createStore, del, get, set } from "idb-keyval"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { analyze, ApiError, sendJson, setAccount, setStudySession } from "@/lib/api"; +import { isSavedNotebook, loadNotebook, writeNotebook, type SavedNotebook } from "@/lib/persist"; +import { AnalysisScheduler, RequestTimedOut } from "@/lib/scheduler"; +import { NotebookStore } from "@/lib/store"; +import type { AnalyzeResponse, PenStroke } from "@/lib/types"; + +const ink: PenStroke = { + id: "s1", color: "#000", width: 2, createdAt: 1, + points: [{ x: 1, y: 2, timestamp: 1, pressure: .5 }], +}; +const valid = (): SavedNotebook => ({ strokes: [structuredClone(ink)], version: 1 }); +const db = createStore("quack", "notes"); +let sequence = 0; +const key = () => `release-safety-${sequence++}`; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + vi.useRealTimers(); + setAccount(async () => null, () => {}); + setStudySession(() => null); +}); + +describe("saved notebook integrity", () => { + const invalid: [string, unknown][] = [ + ["null", null], ["non-object", []], ["missing version", { strokes: [] }], + ["negative version", { ...valid(), version: -1 }], + ["fractional version", { ...valid(), version: 1.5 }], + ["unsafe version", { ...valid(), version: Number.MAX_SAFE_INTEGER + 1 }], + ["invalid strokes", { version: 1, strokes: [null] }], + ["empty stroke", { version: 1, strokes: [{ ...ink, points: [] }] }], + ["duplicate ids", { version: 1, strokes: [ink, ink] }], + ["invalid coordinates", { version: 1, strokes: [{ ...ink, points: [{ x: NaN, y: 1, timestamp: 0 }] }] }], + ["infinite width", { version: 1, strokes: [{ ...ink, width: Infinity }] }], + ["negative width", { version: 1, strokes: [{ ...ink, width: -1 }] }], + ["missing timestamp", { version: 1, strokes: [{ ...ink, points: [{ x: 1, y: 1 }] }] }], + ["invalid pressure", { version: 1, strokes: [{ ...ink, points: [{ x: 1, y: 1, timestamp: 1, pressure: 2 }] }] }], + ["reversed bounds", { ...valid(), dirty: [{ minX: 10, maxX: 0, minY: 0, maxY: 10 }] }], + ["invalid seen record", { ...valid(), seen: { m: { dismissed: "yes", area: null } } }], + ]; + + it.each(invalid)("preserves a corrupt value: %s", async (_label, stored) => { + const name = key(); + await set(name, stored, db); + expect(isSavedNotebook(stored)).toBe(false); + expect(await loadNotebook(name)).toEqual({ status: "failed" }); + expect(await writeNotebook(valid(), null, name)).toBe("conflict"); + expect(await get(name, db)).toEqual(stored); + }); + + it("accepts older notebooks without optional metadata", async () => { + const name = key(); + await set(name, valid(), db); + expect(await loadNotebook(name)).toEqual({ status: "found", saved: valid() }); + }); + + it("reports a genuinely absent save as empty", async () => { + expect(await loadNotebook(key())).toEqual({ status: "empty" }); + }); + + it("does not resurrect notes deleted by another tab", async () => { + const name = key(); + await set(name, valid(), db); + await del(name, db); + expect(await writeNotebook({ ...valid(), version: 2 }, 1, name)).toBe("conflict"); + expect(await get(name, db)).toBeUndefined(); + }); + + it("allows exactly one of two concurrent first writers", async () => { + const name = key(); + const results = await Promise.all([ + writeNotebook(valid(), null, name), + writeNotebook({ strokes: [], version: 2 }, null, name), + ]); + expect(results.sort()).toEqual(["conflict", "saved"]); + }); + + it("rejects invalid writes before they replace good notes", async () => { + const name = key(); + await set(name, valid(), db); + expect(await writeNotebook({ ...valid(), version: NaN }, 1, name)).toBe("failed"); + expect(await writeNotebook({ ...valid(), version: 0 }, 1, name)).toBe("failed"); + expect(await get(name, db)).toEqual(valid()); + }); + + it("keeps saving disabled when hydration found corruption", async () => { + const name = key(); + const corrupted = { strokes: "recover this", version: 1 }; + await set(name, corrupted, db); + const notebook = new NotebookStore({ key: name }); + expect((await notebook.hydrate()).status).toBe("failed"); + notebook.add(ink); + notebook.flush(); + expect(await get(name, db)).toEqual(corrupted); + }); +}); + +describe("HTTP failures and account isolation", () => { + it.each([400, 401, 403, 413, 422, 429, 500, 503])("rejects HTTP %i without accepting its body", async (status) => { + const signedOut = vi.fn(); + setAccount(async () => "token", signedOut); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("private server error", { status }))); + await expect(sendJson("GET", "/api/test")).rejects.toMatchObject({ status }); + expect(signedOut).toHaveBeenCalledTimes(status === 401 ? 1 : 0); + }); + + it("passes the abort signal and signed-in token to fetch", async () => { + const controller = new AbortController(); + const fetchMock = vi.fn().mockResolvedValue(new Response('{"ok":true}')); + vi.stubGlobal("fetch", fetchMock); + setAccount(async () => "test-token", () => {}); + await sendJson("POST", "/api/test", { value: 1 }, controller.signal); + expect(fetchMock).toHaveBeenCalledWith("/api/test", expect.objectContaining({ + signal: controller.signal, body: '{"value":1}', + headers: { authorization: "Bearer test-token", "content-type": "application/json" }, + })); + }); + + it("does not send an old token after sign-out", async () => { + const fetchMock = vi.fn().mockImplementation(async () => new Response("{}")); + vi.stubGlobal("fetch", fetchMock); + setAccount(async () => "old-token", () => {}); + await sendJson("GET", "/api/test"); + setAccount(async () => null, () => {}); + await sendJson("GET", "/api/test"); + expect(fetchMock.mock.calls[1][1].headers).toEqual({}); + }); + + it("includes the current study session in analysis requests", async () => { + const fetchMock = vi.fn().mockResolvedValue(new Response("{}")); + vi.stubGlobal("fetch", fetchMock); + setStudySession(() => "study-1"); + await analyze("ink-session", 3, "data:image/png;base64,", "manual", ["m1"]); + expect(JSON.parse(fetchMock.mock.calls[0][1].body)).toMatchObject({ studySessionId: "study-1", snapshotVersion: 3 }); + }); + + it("returns a typed error without echoing sensitive response content", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("secret", { status: 500 }))); + await expect(sendJson("GET", "/api/test")).rejects.toBeInstanceOf(ApiError); + await expect(sendJson("GET", "/api/test")).rejects.not.toThrow("secret"); + }); +}); + +describe("scheduler cancellation and lifecycle", () => { + function setup() { + vi.useFakeTimers(); + const deliveries: ((response: AnalyzeResponse) => void)[] = []; + const send = vi.fn(() => + new Promise((resolve) => deliveries.push(resolve))); + const restore = vi.fn(); + const onResult = vi.fn(); + const onError = vi.fn(); + const onBusyChange = vi.fn(); + const scheduler = new AnalysisScheduler({ + getVersion: () => 1, + capture: () => ({ hash: "ink", version: 1, image: "data:", restore }), + send, onResult, onError, onBusyChange, requestTimeoutMs: 100, + }); + const result = { snapshotVersion: 1, stale: false, modelIssue: null } as AnalyzeResponse; + return { scheduler, send, restore, onResult, onError, onBusyChange, deliveries, result }; + } + + it("times out even if the transport ignores abort, then retries", async () => { + const h = setup(); + h.scheduler.checkNow(); + await vi.advanceTimersByTimeAsync(100); + expect(h.onError).toHaveBeenCalledWith(expect.any(RequestTimedOut), "manual"); + expect(h.restore).toHaveBeenCalledOnce(); + expect(h.onBusyChange).toHaveBeenLastCalledWith(false); + await vi.advanceTimersByTimeAsync(2000); + expect(h.send).toHaveBeenCalledTimes(2); + h.scheduler.dispose(); + await vi.advanceTimersByTimeAsync(0); + }); + + it("does not apply a late response after disposal", async () => { + const h = setup(); + h.scheduler.checkNow(); + h.scheduler.dispose(); + h.deliveries[0](h.result); + await vi.advanceTimersByTimeAsync(10_000); + expect(h.onResult).not.toHaveBeenCalled(); + expect(h.onError).not.toHaveBeenCalled(); + expect(h.send).toHaveBeenCalledOnce(); + expect(h.onBusyChange).toHaveBeenLastCalledWith(false); + }); + + it("cannot be restarted after disposal", async () => { + const h = setup(); + h.scheduler.dispose(); + h.scheduler.checkNow(); + h.scheduler.notifyEdit(); + h.scheduler.setEnabled(true); + await vi.advanceTimersByTimeAsync(10_000); + expect(h.send).not.toHaveBeenCalled(); + }); + + it("turning monitoring off clears a queued manual request", async () => { + const h = setup(); + h.scheduler.notifyEdit(); + await vi.advanceTimersByTimeAsync(800); + h.scheduler.checkNow(); + h.scheduler.setEnabled(false); + await vi.advanceTimersByTimeAsync(10_000); + expect(h.send).toHaveBeenCalledOnce(); + expect(h.onError).not.toHaveBeenCalled(); + h.scheduler.dispose(); + }); + + it("re-enabling immediately after cancellation starts one new run", async () => { + const h = setup(); + h.scheduler.checkNow(); + h.scheduler.setEnabled(false); + h.scheduler.setEnabled(true); + await vi.advanceTimersByTimeAsync(0); + expect(h.send).toHaveBeenCalledTimes(2); + h.deliveries[0](h.result); + await vi.advanceTimersByTimeAsync(0); + expect(h.onResult).not.toHaveBeenCalled(); + h.deliveries[1](h.result); + await vi.advanceTimersByTimeAsync(0); + expect(h.onResult).toHaveBeenCalledOnce(); + h.scheduler.dispose(); + }); +}); diff --git a/apps/web/vitest.config.mts b/apps/web/vitest.config.mts index 62ddde1..954a34d 100644 --- a/apps/web/vitest.config.mts +++ b/apps/web/vitest.config.mts @@ -3,5 +3,5 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ resolve: { alias: { "@": fileURLToPath(new URL(".", import.meta.url)) } }, - test: { environment: "jsdom", include: ["tests/unit/**/*.test.ts"] }, + test: { allowOnly: false, environment: "jsdom", include: ["tests/unit/**/*.test.ts"] }, }); diff --git a/docs/RELEASE_SAFETY.md b/docs/RELEASE_SAFETY.md new file mode 100644 index 0000000..878b04e --- /dev/null +++ b/docs/RELEASE_SAFETY.md @@ -0,0 +1,118 @@ +# Release verification and safeguards + +## Reproduce the checks + +Use Python 3.12, Node 22, PostgreSQL 18 and Gitleaks 8.30.1. From a clean checkout: + +```bash +python3.12 -m venv services/api/.venv +services/api/.venv/bin/python -m pip install -r services/api/requirements.lock -r services/api/requirements-tooling.lock -r services/api/requirements-dev.txt +npm ci --prefix apps/web +(cd apps/web && npx playwright install --with-deps chromium) +# Create an empty LOCAL disposable database named quack_ledger_test. +export QUACK_LEDGER_TEST_DATABASE_URL=postgresql://localhost/quack_ledger_test +npm run verify:release +``` + +The database role needs permission to create a test role for the row-security regression. +Ledger tests truncate only their disposable local database. Never supply production credentials. +`QUACK_TEST_PYTHON` and `QUACK_GITLEAKS` can select explicit tooling paths. +`QUACK_E2E_PORT_SHIFT` shifts the browser suite's ports if other development servers are running. + +The command checks dependency consistency, locked inputs, focused tests, the public-file allowlist, +reachable publication history, staged and historical secrets, Ruff, mypy, offline backend tests, +ESLint, TypeScript, frontend units, the production build, compiled browser checks and all existing +browser journeys. It refuses skipped backend tests and unknown pytest settings or markers. +The two paid live tests are deliberately deselected. No model credits are needed. +Historical prompt comparisons use a classification-only fixture checked against the published +synthetic dataset. It contains no raw model text, usage records, images or private evaluation paths. + +Production credentials and API routing overrides are removed from the verification subprocess +environment. Browser journeys use isolated temporary storage, local account stand-ins and a fake +model endpoint. The compiled suite stubs API responses inside the browser. Production debug-tool +restrictions remain covered by the existing compiled tests. + +Run advisory checks separately (they need advisory-service network access): + +```bash +services/api/.venv/bin/python -m pip_audit --no-deps --disable-pip -r services/api/requirements.lock -r services/api/requirements-tooling.lock +npm audit --prefix apps/web +``` + +The workflow runs these audits and the complete release command on pull requests and pushes to +`main`, plus manual dispatch. One GitHub-hosted job avoids duplicate branch/PR browser runs and +uses an ephemeral PostgreSQL service. Chromium is installed explicitly; browser traces and +screenshots are retained for seven days after failures. Focused Playwright and Vitest tests fail +locally and in CI. An additional source guard rejects committed `test.only` forms. + +Local evidence does not establish hosted CI or deployment health. Check the workflow attached +to the exact reviewed commit before merging. Advisory results are point-in-time observations. + +## Accounting: production versus local development + +Production uses **PostgreSQL-backed accounting**. Before a model operation, an advisory transaction +lock coordinates independent connections while allocating a durable debit for the complete retry +allowance. The router's attempts share that debit; they do not multiply it again. Account scope +ignores rotating client session ids. Project, account, request and hourly ceilings remain enforced. +Successful operations settle only when every usage receipt is complete and successful. Unknown +usage, timeout, failed retry, cancellation and crash retain the conservative reservation indefinitely. +Cancelled requests also write a usage receipt. Database failures prevent further model operations; +invalid receipts retain the debit and stop subsequent spending. No production spending records +are deleted or reset by verification or deployment. + +Local development can use the append-only JSON file ledger. Each retry independently checks +committed spend and reserves its next worst-case cost. Replay validates every incoming record +before adding any of them. Invalid costs, timestamps or session ids fail closed. Write failures +count completed usage in memory and block further spending for that process. Hourly accounting +handles timestamps received out of order. These disk and replay safeguards apply to the file +backend, not to the PostgreSQL implementation. File reservations coordinate only one process; +file accounting must not replace the production transactional ledger. + +Retain one production Modal container: notebook analysis sessions, rate windows and other state +remain process-local even though production spending is coordinated by PostgreSQL. Existing +inference quotas are deployment configuration, not test defaults. Do not increase them for a +release. Provider cost estimates include a fixed input allowance and are not provider billing caps. + +## Student-facing safety + +Handwriting recognition proposes text; the restricted parser and deterministic SymPy verifier +establish mathematical evidence. Runtime response validation rejects unverified or duplicate +mistakes, inconsistent selected mistakes, present/resolved overlap, and alerts without a current +new verified mistake. These checks run with Python assertions disabled and do not turn model +confidence into mathematical proof. + +Notebook hydration validates versions, unique stroke ids, finite geometry, points and metadata. +Unreadable or corrupt values remain stored and new writing cannot overwrite them. Cross-tab +version transactions prevent simultaneous first writers and recreation after another tab deletes +notes. Export, deletion and per-account storage separation retain their existing behavior. + +The production scheduler still supports checks of newer ink while an older check runs. Aborted +transports cannot deliver late warnings; a watchdog releases checks even if the transport ignores +abort. Cancelled captures finish cleanup before replacement work starts. Existing stale-reply, +account, mastery and practice regressions cover the broader interaction. + +Bricolage Grotesque and Figtree are bundled as Latin WOFF2 assets with the existing +[SIL font licenses](licenses/THIRD_PARTY_NOTICES.txt). Their names, weights and CSS variables are +preserved. Builds no longer fetch Google Fonts; compiled browser tests verify same-origin font +requests, loaded faces, responsive rendering and saved-ink recovery. + +## HTTP diagnostics + +Every response includes a fresh server-generated `X-Request-ID`. Configured CORS origins can read +it. Nested application logs carry the same context. HTTP events contain the route template, +method, status and duration, without literal student paths, query strings, request bodies or auth +headers. Unhandled failures return a generic response; logs contain the exception class and code +locations, not exception messages, source code, handwriting or local variables. API responses use +`Cache-Control: no-store`. The correlation id is independent of the analysis payload's `requestId`. + +The Next.js rewrite forwards backend response headers through the same-origin frontend proxy. +Vercel and Modal generate their own platform request identifiers; those are separate from Quack's. + +## Limits of the evidence + +Synthetic and mocked checks do not establish real handwriting accuracy, physical Apple Pencil or +iPad behavior, production Supabase registration/deletion/recovery flows, personalized recognition +quality, or provider billing. A release must independently inspect actual hosted CI, frontend +and backend source versions, HTTPS, health/readiness, auth guards, proxy behavior and debug-route +restrictions. Deploying the frontend does not deploy Modal. A live paid inference test requires +explicit release-specific authorization. Never exercise destructive database tests in production. diff --git a/package.json b/package.json new file mode 100644 index 0000000..eae8a24 --- /dev/null +++ b/package.json @@ -0,0 +1,8 @@ +{ + "name": "quack", + "private": true, + "scripts": { + "verify": "python3 scripts/verify.py", + "verify:release": "python3 scripts/verify.py --build --browser" + } +} diff --git a/scripts/check_publication.py b/scripts/check_publication.py index af461f0..f2b5526 100755 --- a/scripts/check_publication.py +++ b/scripts/check_publication.py @@ -19,6 +19,7 @@ "docs/ARCHITECTURE.md", "docs/SETUP.md", "docs/TESTING.md", + "docs/RELEASE_SAFETY.md", "docs/PUBLICATION.md", "docs/CREDITS.md", "docs/hackathon/JUDGE_GUIDE.md", diff --git a/scripts/check_release.py b/scripts/check_release.py new file mode 100644 index 0000000..0d58e22 --- /dev/null +++ b/scripts/check_release.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Validate locked release inputs, published files and committed test selection.""" + +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def main() -> int: + errors: list[str] = [] + names = subprocess.check_output(["git", "ls-files", "-z"], cwd=ROOT).decode().split("\0") + for name in filter(None, names): + path = ROOT / name + if not path.is_file(): + errors.append(f"Missing published file: {name}") + continue + if "/tests/" in name and path.suffix in {".ts", ".tsx", ".py"}: + if re.search(r"\b(?:test|it|describe)\s*\.\s*only\s*[(<]", path.read_text()): + errors.append(f"Focused test committed: {name}") + package = json.loads((ROOT / "apps/web/package.json").read_text()) + locked = json.loads((ROOT / "apps/web/package-lock.json").read_text())["packages"][""] + for group in ("dependencies", "devDependencies"): + if package[group] != locked[group]: + errors.append(f"Web {group} differ from lockfile") + for name in ("requirements.lock", "requirements-tooling.lock"): + for line in (ROOT / "services/api" / name).read_text().splitlines(): + if line.strip() and not line.startswith("#") and not re.fullmatch(r"[\w.-]+==[\w.+-]+", line): + errors.append(f"Unpinned Python input: {name}") + if "next/font/google" in (ROOT / "apps/web/app/layout.tsx").read_text(): + errors.append("Build must use bundled fonts") + for error in errors: + print(error) + print(f"Release inputs: {len(errors)} errors") + return bool(errors) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/public-files.txt b/scripts/public-files.txt index 10cc22e..b636bb4 100644 --- a/scripts/public-files.txt +++ b/scripts/public-files.txt @@ -427,3 +427,16 @@ docs/media/hint-detail.webp docs/media/correction-detail.webp docs/media/practice-detail.webp docs/media/learning-detail.webp +.github/workflows/verify.yml +apps/web/app/fonts/bricolage-grotesque-latin-700.woff2 +apps/web/app/fonts/figtree-latin-variable.woff2 +apps/web/tests/production/safety.spec.ts +apps/web/tests/unit/release-safety.test.ts +docs/RELEASE_SAFETY.md +package.json +scripts/check_release.py +scripts/verify.py +services/api/quack/observability.py +services/api/requirements-tooling.lock +services/api/tests/test_release_safety.py +services/api/tests/fixtures/recorded-routing.json diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 0000000..2c3e754 --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Run Quack's release checks. No live model calls, deployments or production credentials.""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def test_environment() -> dict[str, str]: + """Never inherit production keys, databases, routes or budgets into release checks.""" + keep = {"QUACK_TEST_PYTHON", "QUACK_LEDGER_TEST_DATABASE_URL", "QUACK_E2E_PORT_SHIFT", "QUACK_GITLEAKS"} + env = { + key: value for key, value in os.environ.items() + if not key.startswith(("QUACK_", "MODEL_", "SUPABASE_", "NEXT_PUBLIC_")) or key in keep + } + env.pop("API_URL", None) + env.update(CI="true", QUACK_ENV="production", MODEL_API_KEY="", + QUACK_RELEASE_VERIFY="1", NEXT_TELEMETRY_DISABLED="1") + return env + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--browser", action="store_true", help="also run all mocked Playwright journeys") + parser.add_argument("--build", action="store_true", help="also compile the production web app") + args = parser.parse_args() + env = test_environment() + if not env.get("QUACK_LEDGER_TEST_DATABASE_URL"): + parser.error("set QUACK_LEDGER_TEST_DATABASE_URL to a disposable local quack_ledger_test database") + gitleaks = env.get("QUACK_GITLEAKS") or shutil.which("gitleaks") + if not gitleaks: + parser.error("install Gitleaks 8.30.1 or set QUACK_GITLEAKS to its executable") + local_python = ROOT / "services/api/.venv/bin/python" + python = os.environ.get("QUACK_TEST_PYTHON") or (str(local_python) if local_python.exists() else sys.executable) + checks = [ + ("Dependency consistency", ".", [python, "-m", "pip", "check"]), + ("Release inputs and focused-test guard", ".", [python, "scripts/check_release.py"]), + ("Public file and history policy", ".", [python, "scripts/check_publication.py", "--history"]), + ("History secret scan", ".", [gitleaks, "git", "--redact", "--no-banner", "--log-opts=--all"]), + ("Staged secret scan", ".", [gitleaks, "git", "--redact", "--no-banner", "--pre-commit", "--staged"]), + ("Python lint", "services/api", [python, "-m", "ruff", "check", "quack", "tests"]), + ("Python types", "services/api", [python, "-m", "mypy", "quack"]), + ("Backend regressions", "services/api", [python, "-m", "pytest", "-q", "-m", "not live"]), + ("Web lint", "apps/web", ["npm", "run", "lint"]), + ("Web types", "apps/web", ["npm", "run", "typecheck"]), + ("Web logic", "apps/web", ["npm", "test"]), + ] + if args.build: + checks.append(("Production build", "apps/web", ["npm", "run", "build"])) + if args.browser: + checks.append(("Compiled app safety", "apps/web", ["npm", "run", "test:production"])) + if args.browser: + checks.append(("Browser journeys (fake model)", "apps/web", ["npm", "run", "test:e2e"])) + for label, directory, command in checks: + print(f"\n{label}", flush=True) + result = subprocess.run(command, cwd=ROOT / directory, env=env, check=False) + if result.returncode: + print(f"\nFAILED: {label} (exit {result.returncode})", file=sys.stderr) + return result.returncode + print(f"\nPassed all {len(checks)} checks.", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/api/pyproject.toml b/services/api/pyproject.toml index 5362729..a835831 100644 --- a/services/api/pyproject.toml +++ b/services/api/pyproject.toml @@ -9,7 +9,7 @@ pythonpath = ["."] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" markers = ["live: calls the real Meta Model API and spends credits; needs MODEL_API_KEY"] -addopts = "-m 'not live'" +addopts = "-m 'not live' --strict-markers --strict-config" [tool.ruff] line-length = 125 diff --git a/services/api/quack/account_api.py b/services/api/quack/account_api.py index ebcab23..7df771c 100644 --- a/services/api/quack/account_api.py +++ b/services/api/quack/account_api.py @@ -43,6 +43,7 @@ PracticeReveal, PracticeStart, PracticeSummary, + PracticeTopic, PracticeTopicOption, Profile, Recap, @@ -218,14 +219,16 @@ def _owned(store: UserStore, user_id: str, session_id: str | None) -> str | None return session_id if session_id and store.session(user_id, session_id) else None -def _general_topic(store: UserStore, user_id: str, origin: str | None) -> str: +def _general_topic(store: UserStore, user_id: str, origin: str | None) -> PracticeTopic: """What the recap recommended, if it did, and otherwise simply the first one on the list.""" session = store.session(user_id, origin) if origin else None if session and session["recap_json"]: recommendation = Recap.model_validate_json(session["recap_json"]).recommendation if recommendation: - return recommendation.topic + for topic in practice_service.SUPPORTED: + if topic == recommendation.topic: + return topic return practice_service.SUPPORTED[0] @@ -498,7 +501,11 @@ async def practice_topics() -> PracticeOptions: @router.post("/practice/sessions") async def start_practice(body: PracticeStart, identity: Identity = you) -> PracticeProblem: user_id = identity.user_id - topic = body.topic or await asyncio.to_thread(_general_topic, store, user_id, body.origin_session_id) + topic: PracticeTopic + if body.topic is not None: + topic = body.topic + else: + topic = await asyncio.to_thread(_general_topic, store, user_id, body.origin_session_id) origin = await asyncio.to_thread(_owned, store, user_id, body.origin_session_id) session = await asyncio.to_thread(store.new_practice_session, user_id, origin, topic) # Only one of the student's own. Anything else simply isn't linked. diff --git a/services/api/quack/config.py b/services/api/quack/config.py index e302ac1..f74a4f3 100644 --- a/services/api/quack/config.py +++ b/services/api/quack/config.py @@ -15,18 +15,19 @@ class Settings(BaseSettings): model_config = SettingsConfigDict( - env_file=(REPO_ROOT / ".env", API_ROOT / ".env"), extra="ignore", populate_by_name=True + env_file=(REPO_ROOT / ".env", API_ROOT / ".env"), extra="ignore", populate_by_name=True, + allow_inf_nan=False, ) model_api_key: str = Field(default="", alias="MODEL_API_KEY", repr=False) model_api_base_url: str = Field(default="https://api.meta.ai/v1", alias="MODEL_API_BASE_URL") # Development-only routes stay hidden unless this is set on purpose. - env: str = Field(default="production", alias="QUACK_ENV") + env: Literal["production", "development"] = Field(default="production", alias="QUACK_ENV") cors_origins: str = Field(default="http://localhost:3000", alias="QUACK_CORS_ORIGINS") - rate_limit_per_minute: int = Field(default=60, alias="QUACK_RATE_LIMIT_PER_MINUTE") + rate_limit_per_minute: int = Field(default=60, gt=0, alias="QUACK_RATE_LIMIT_PER_MINUTE") # Client addresses can be forged through a proxy, so this cap is what actually bounds spend rate. - global_rate_limit_per_minute: int = Field(default=300, alias="QUACK_GLOBAL_RATE_LIMIT_PER_MINUTE") + global_rate_limit_per_minute: int = Field(default=300, gt=0, alias="QUACK_GLOBAL_RATE_LIMIT_PER_MINUTE") # JSON object of per-purpose overrides, for example: # {"transcribe": {"model": "muse-wonder-1.0", "reasoning_effort": "medium"}} @@ -40,21 +41,21 @@ class Settings(BaseSettings): confirm_in_parallel: bool = Field(default=True, alias="QUACK_CONFIRM_IN_PARALLEL") # How long the second reading can take before another goes alongside it. Most finish in 6 s, and # those that use up all their tokens take 19 s. See `AnalysisService._second_reading`. - second_reading_patience_s: float = Field(default=14.0, alias="QUACK_SECOND_READING_PATIENCE_S") + second_reading_patience_s: float = Field(default=14.0, gt=0, alias="QUACK_SECOND_READING_PATIENCE_S") # Re-read when the fast reading was unclear. Off until evaluation shows it pays for itself. second_reading_on_uncertain: bool = Field(default=False, alias="QUACK_SECOND_READING_ON_UNCERTAIN") - reading_cache_size: int = Field(default=256, alias="QUACK_READING_CACHE_SIZE") + reading_cache_size: int = Field(default=256, ge=0, alias="QUACK_READING_CACHE_SIZE") # Lines read and agreed on, across sessions, so unchanged lines needn't be read again. - line_memory_size: int = Field(default=20_000, alias="QUACK_LINE_MEMORY_SIZE") - verify_timeout_s: float = Field(default=4.0, alias="QUACK_VERIFY_TIMEOUT_S") + line_memory_size: int = Field(default=20_000, ge=0, alias="QUACK_LINE_MEMORY_SIZE") + verify_timeout_s: float = Field(default=4.0, gt=0, alias="QUACK_VERIFY_TIMEOUT_S") # Credit guard rails. The project ceiling leaves US$15 of the US$150 allocation in reserve. - budget_project_usd: float = Field(default=135.0, alias="QUACK_BUDGET_PROJECT_USD") - budget_session_usd: float = Field(default=2.0, alias="QUACK_BUDGET_SESSION_USD") - budget_request_usd: float = Field(default=0.50, alias="QUACK_BUDGET_REQUEST_USD") + budget_project_usd: float = Field(default=135.0, ge=0, alias="QUACK_BUDGET_PROJECT_USD") + budget_session_usd: float = Field(default=2.0, ge=0, alias="QUACK_BUDGET_SESSION_USD") + budget_request_usd: float = Field(default=0.50, ge=0, alias="QUACK_BUDGET_REQUEST_USD") # A session id is chosen by the client, so per-session limits only stop a runaway tab. # This one stops anyone draining the credits quickly, whatever ids they use. - budget_hourly_usd: float = Field(default=5.0, alias="QUACK_BUDGET_HOURLY_USD") + budget_hourly_usd: float = Field(default=5.0, ge=0, alias="QUACK_BUDGET_HOURLY_USD") ledger_path: Path = Field(default=API_ROOT / "var" / "usage.jsonl", alias="QUACK_LEDGER_PATH") ledger_backend: Literal["file", "postgres"] = Field(default="file", alias="QUACK_LEDGER_BACKEND") diff --git a/services/api/quack/llm/ledger.py b/services/api/quack/llm/ledger.py index c14f5f4..145a45a 100644 --- a/services/api/quack/llm/ledger.py +++ b/services/api/quack/llm/ledger.py @@ -3,14 +3,34 @@ from __future__ import annotations import json +import logging +import math import threading import time from collections import defaultdict, deque from collections.abc import Iterator -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager, nullcontext from dataclasses import asdict, dataclass, field from pathlib import Path +log = logging.getLogger("quack.ledger") + + +def _nonnegative(value: object) -> float: + if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value) or value < 0: + raise ValueError("ledger amounts must be finite, nonnegative numbers") + return float(value) + + +def _validated(record: object) -> dict: + if not isinstance(record, dict): + raise ValueError("ledger record must be an object") + _nonnegative(record.get("estimated_usd", 0.0)) + _nonnegative(record.get("timestamp", time.time())) + if not isinstance(record.get("session_id", ""), str): + raise ValueError("ledger session must be text") + return record + class BudgetExceeded(Exception): def __init__(self, scope: str, spent: float, limit: float) -> None: @@ -51,6 +71,9 @@ def __init__( hourly_limit: float | None = None, ) -> None: self.path = path + for limit in (project_limit, session_limit, request_limit, hourly_limit): + if limit is not None: + _nonnegative(limit) self.project_limit = project_limit self.session_limit = session_limit self.request_limit = request_limit @@ -63,6 +86,7 @@ def __init__( self._reserved_project = 0.0 self._reserved_session: dict[str, float] = defaultdict(float) self._read_up_to = 0 + self._write_failed = False if path is not None and path.exists(): self._repair_tail(path) self._catch_up() @@ -102,7 +126,7 @@ def _catch_up(self) -> None: data = handle.read() complete = data.rfind(b"\n") + 1 # All or nothing, or a retry after a bad line would count the good ones twice. - records = [json.loads(line) for line in data[:complete].splitlines() if line.strip()] + records = [_validated(json.loads(line)) for line in data[:complete].splitlines() if line.strip()] for record in records: self._add(record) self._read_up_to += complete @@ -117,8 +141,8 @@ def _add(self, record: dict) -> None: def _hourly_spend(self) -> float: cutoff = time.time() - 3600 - while self._recent and self._recent[0][0] < cutoff: - self._recent.popleft() + # Other processes and wall-clock changes can append records out of timestamp order. + self._recent = deque((stamp, cost) for stamp, cost in self._recent if stamp >= cutoff) return sum(cost for _, cost in self._recent) def session_spend(self, session_id: str) -> float: @@ -126,9 +150,12 @@ def session_spend(self, session_id: str) -> float: return self._session_spend.get(session_id, 0.0) def _check_locked(self, session_id: str, worst_case_usd: float) -> None: + _nonnegative(worst_case_usd) + if self._write_failed: + raise BudgetExceeded("ledger", self.project_spend, self.project_limit) try: self._catch_up() - except json.JSONDecodeError as exc: + except (ValueError, OSError) as exc: # We can't tell what has been spent, so nothing more gets spent. raise BudgetExceeded("ledger", self._project_spend, self.project_limit) from exc if worst_case_usd > self.request_limit: @@ -151,6 +178,13 @@ def check(self, session_id: str, worst_case_usd: float) -> None: with self._lock: self._check_locked(session_id, worst_case_usd) + def reserve_operation(self, session_id: str, worst_case_usd: float) -> AbstractContextManager[None]: + """File accounting reserves each attempt instead of prepaying retries.""" + return nullcontext() + + def reserve_attempt(self, session_id: str, worst_case_usd: float) -> AbstractContextManager[None]: + return self.reserve(session_id, worst_case_usd) + @contextmanager def reserve(self, session_id: str, worst_case_usd: float) -> Iterator[None]: """Hold a request's worst-case cost against the limits until it has finished.""" @@ -171,21 +205,23 @@ def reserve(self, session_id: str, worst_case_usd: float) -> Iterator[None]: del self._reserved_session[session_id] def record(self, record: UsageRecord) -> None: - payload = asdict(record) + payload = _validated(asdict(record)) with self._lock: if self.path is None: self._add(payload) return - self.path.parent.mkdir(parents=True, exist_ok=True) try: + self.path.parent.mkdir(parents=True, exist_ok=True) # First as well, so that a file replaced since the last look is noticed. self._catch_up() with self.path.open("a") as handle: handle.write(json.dumps(payload) + "\n") self._catch_up() - except json.JSONDecodeError: + except (ValueError, OSError): # The call has already happened, so count it, and the next check will refuse. self._add(payload) + self._write_failed = True + log.error("ledger_write_failed") def read_records(path: Path) -> list[UsageRecord]: diff --git a/services/api/quack/llm/postgres_ledger.py b/services/api/quack/llm/postgres_ledger.py index 2d69437..1b154ac 100644 --- a/services/api/quack/llm/postgres_ledger.py +++ b/services/api/quack/llm/postgres_ledger.py @@ -11,7 +11,7 @@ import time import uuid from collections.abc import Iterator -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager, nullcontext from contextvars import ContextVar from dataclasses import asdict from decimal import Decimal @@ -20,7 +20,7 @@ from psycopg.types.json import Jsonb from quack.db.database import Database -from quack.llm.ledger import BudgetExceeded, Ledger, UsageRecord +from quack.llm.ledger import BudgetExceeded, Ledger, UsageRecord, _nonnegative, _validated _active_debit: ContextVar[str | None] = ContextVar("quack_model_debit", default=None) _LOCK_ID = 743_812_095 @@ -36,8 +36,8 @@ def __init__( if not url: raise ValueError("PostgreSQL accounting requires QUACK_DATABASE_URL") for limit in (project_limit, session_limit, request_limit, hourly_limit): - if limit is not None and (not math.isfinite(limit) or limit < 0): - raise ValueError("model budgets must be finite and nonnegative") + if limit is not None: + _nonnegative(limit) self.path = None self.project_limit = project_limit self.session_limit = session_limit @@ -124,10 +124,23 @@ def _allocate(self, session_id: str, amount: float, *, debit: bool) -> str: def check(self, session_id: str, worst_case_usd: float) -> None: self._allocate(session_id, worst_case_usd, debit=False) + def reserve_operation(self, session_id: str, worst_case_usd: float) -> AbstractContextManager[None]: + """Keep the production operation's complete retry allowance durably prepaid.""" + return self.reserve(session_id, worst_case_usd) + + def reserve_attempt(self, session_id: str, worst_case_usd: float) -> AbstractContextManager[None]: + # Each attempt is already covered by the surrounding durable operation debit. + if _active_debit.get() is None or not self._healthy: + raise BudgetExceeded("ledger", 0, self.project_limit) + _nonnegative(worst_case_usd) + return nullcontext() + @contextmanager def reserve(self, session_id: str, worst_case_usd: float) -> Iterator[None]: from quack.llm.router import MAX_ATTEMPTS + _nonnegative(worst_case_usd) + # The existing router can retry an operation. Prepay its complete retry # allowance. Voice has no retry but uses the same conservative bound. # Never expire an unresolved debit. @@ -162,6 +175,11 @@ def _settle_known(self, debit_id: str) -> None: def record(self, record: UsageRecord) -> None: debit_id = _active_debit.get() + try: + _validated(asdict(record)) + except ValueError as exc: + self._healthy = False + raise BudgetExceeded("ledger", 0, self.project_limit) from exc if debit_id is None or not math.isfinite(record.estimated_usd) or record.estimated_usd < 0: raise BudgetExceeded("ledger", 0, self.project_limit) try: diff --git a/services/api/quack/llm/router.py b/services/api/quack/llm/router.py index 74d52ea..47542c3 100644 --- a/services/api/quack/llm/router.py +++ b/services/api/quack/llm/router.py @@ -8,14 +8,18 @@ import asyncio import json import logging +import math import random import time from dataclasses import dataclass, replace from enum import StrEnum -from typing import Any, Generic, TypeVar +from typing import Any, Generic, TypeVar, cast import openai from openai import AsyncOpenAI +from openai.types.chat import ChatCompletionMessageParam +from openai.types.shared import ReasoningEffort +from openai.types.shared_params import ResponseFormatJSONSchema from pydantic import BaseModel, ValidationError from quack.config import Settings @@ -49,6 +53,18 @@ class ModelRoute: # A student is waiting. After a timeout, the next pause tries again sooner than a retry would end. retry_timeouts: bool = True + def __post_init__(self) -> None: + if self.model not in ALLOWED_TOKEN_MODELS: + raise ValueError("model has no known price") + if self.reasoning_effort not in ("none", "minimal", "low", "medium", "high"): + raise ValueError("invalid reasoning effort") + if isinstance(self.timeout_s, bool) or not math.isfinite(self.timeout_s) or self.timeout_s <= 0: + raise ValueError("model timeout must be finite and positive") + if type(self.max_tokens) is not int or self.max_tokens <= 0: + raise ValueError("model token limit must be a positive integer") + if type(self.retry_timeouts) is not bool: + raise ValueError("retry_timeouts must be a boolean") + # Reading is timed per call in docs/IPAD_TEST_ROUND_2.md. The two readings run at different efforts # on purpose: identical ones could share a misreading, and the second is what stops an accusation. @@ -178,7 +194,7 @@ async def complete_json( client = self._get_client() worst_case = worst_case_output_cost(chosen.model, chosen.max_tokens) + ASSUMED_MAX_INPUT_USD try: - with self.ledger.reserve(session_id, worst_case): + with self.ledger.reserve_operation(session_id, worst_case): return await self._send(purpose, messages, schema, chosen, client, session_id, run_id) except BudgetExceeded as exc: log.warning("budget_exceeded", extra={"scope": exc.scope, "purpose": str(purpose)}) @@ -194,107 +210,117 @@ async def _send( session_id: str, run_id: str, ) -> StructuredResult[T]: - response_format = { + response_format: ResponseFormatJSONSchema = { "type": "json_schema", "json_schema": {"name": schema.__name__, "schema": schema.model_json_schema()}, } last_error: ModelError | None = None for attempt in range(1, MAX_ATTEMPTS + 1): - record = UsageRecord( - model=chosen.model, - reasoning_effort=chosen.reasoning_effort, - purpose=str(purpose), - run_id=run_id, - session_id=session_id, - attempt=attempt, - ) - started = time.perf_counter() - try: - response = await client.chat.completions.create( + # Each retry is another billable call. Re-check committed spend and reserve atomically. + worst_case = worst_case_output_cost(chosen.model, chosen.max_tokens) + ASSUMED_MAX_INPUT_USD + with self.ledger.reserve_attempt(session_id, worst_case): + record = UsageRecord( model=chosen.model, - messages=messages, # type: ignore[arg-type] - response_format=response_format, # type: ignore[arg-type] - max_tokens=chosen.max_tokens, - reasoning_effort=chosen.reasoning_effort, # type: ignore[arg-type] - timeout=chosen.timeout_s, - extra_body={"prompt_cache_key": f"quack-{schema.__name__}"}, + reasoning_effort=chosen.reasoning_effort, + purpose=str(purpose), + run_id=run_id, + session_id=session_id, + attempt=attempt, ) - except (openai.APIError, TimeoutError) as exc: - category, retryable, status = _categorise(exc) - record.elapsed_ms = (time.perf_counter() - started) * 1000 - record.http_status = status - record.error_category = category - # The model may have done the work and billed for it, with the reply lost on the way. - # A refused connection never reached it, so that costs nothing. - if isinstance(exc, openai.APITimeoutError | TimeoutError): - record.estimated_usd = worst_case_output_cost(chosen.model, chosen.max_tokens) - self.ledger.record(record) - log.warning("model_request_failed", extra={"category": category, "status": status, "attempt": attempt}) - last_error = ModelError(category, f"model request failed: {category}", status) - if category == "timeout" and not chosen.retry_timeouts: + started = time.perf_counter() + try: + response = await client.chat.completions.create( + model=chosen.model, + messages=cast(list[ChatCompletionMessageParam], messages), + response_format=response_format, + max_tokens=chosen.max_tokens, + reasoning_effort=cast(ReasoningEffort, chosen.reasoning_effort), + timeout=chosen.timeout_s, + extra_body={"prompt_cache_key": f"quack-{schema.__name__}"}, + ) + except asyncio.CancelledError: + # Cancelling our wait doesn't prove the provider stopped its billable work. + record.elapsed_ms = (time.perf_counter() - started) * 1000 + record.error_category = "cancelled" + record.estimated_usd = worst_case + self.ledger.record(record) + log.info("model_request_cancelled", extra={"purpose": str(purpose), "attempt": attempt}) + raise + except (openai.APIError, TimeoutError) as exc: + category, retryable, status = _categorise(exc) + record.elapsed_ms = (time.perf_counter() - started) * 1000 + record.http_status = status + record.error_category = category + # The model may have done the work and billed for it, with the reply lost on the way. + # A refused connection never reached it, so that costs nothing. + if isinstance(exc, openai.APITimeoutError | TimeoutError): + record.estimated_usd = worst_case_output_cost(chosen.model, chosen.max_tokens) + self.ledger.record(record) + log.warning("model_request_failed", extra={"category": category, "status": status, "attempt": attempt}) + last_error = ModelError(category, f"model request failed: {category}", status) + if category == "timeout" and not chosen.retry_timeouts: + raise last_error from exc + if retryable and attempt < MAX_ATTEMPTS: + delay = _retry_delay(attempt, exc) + if delay is not None: + await asyncio.sleep(delay) + continue raise last_error from exc - if retryable and attempt < MAX_ATTEMPTS: - delay = _retry_delay(attempt, exc) - if delay is not None: - await asyncio.sleep(delay) - continue - raise last_error from exc - - record.elapsed_ms = (time.perf_counter() - started) * 1000 - record.http_status = 200 - for key, value in _usage_fields(response.usage).items(): - setattr(record, key, value) - record.estimated_usd = ( - token_cost(chosen.model, record.input_tokens, record.cached_input_tokens, record.output_tokens) - if response.usage is not None - # It says nothing about what it cost. A budget mustn't take that to mean nothing. - else worst_case_output_cost(chosen.model, chosen.max_tokens) - ) - if not response.choices: - record.error_category = "invalid_output" - self.ledger.record(record) - log.warning("model_output_invalid", extra={"attempt": attempt, "purpose": str(purpose)}) - last_error = ModelError("invalid_output", "model response had no choices") - if attempt < MAX_ATTEMPTS - 1: - continue - raise last_error - - choice = response.choices[0] - if choice.finish_reason == "length": - # Reasoning shares the token limit, so a retry at the same limit tends to fail - # the same way and costs the same again. - record.error_category = "truncated" - self.ledger.record(record) - raise ModelError("truncated", "output hit max_tokens before finishing") + record.elapsed_ms = (time.perf_counter() - started) * 1000 + record.http_status = 200 + for key, token_count in _usage_fields(response.usage).items(): + setattr(record, key, token_count) + record.estimated_usd = ( + token_cost(chosen.model, record.input_tokens, record.cached_input_tokens, record.output_tokens) + if response.usage is not None + # It says nothing about what it cost. A budget mustn't take that to mean nothing. + else worst_case_output_cost(chosen.model, chosen.max_tokens) + ) + + if not response.choices: + record.error_category = "invalid_output" + self.ledger.record(record) + log.warning("model_output_invalid", extra={"attempt": attempt, "purpose": str(purpose)}) + last_error = ModelError("invalid_output", "model response had no choices") + if attempt < MAX_ATTEMPTS - 1: + continue + raise last_error + + choice = response.choices[0] + if choice.finish_reason == "length": + # Reasoning shares the token limit, so a retry at the same limit tends to fail + # the same way and costs the same again. + record.error_category = "truncated" + self.ledger.record(record) + raise ModelError("truncated", "output hit max_tokens before finishing") + + try: + value = schema.model_validate_json(choice.message.content or "") + except ValidationError as exc: + record.error_category = "invalid_output" + self.ledger.record(record) + log.warning("model_output_invalid", extra={"attempt": attempt, "purpose": str(purpose)}) + last_error = ModelError("invalid_output", "model output failed schema validation") + if attempt < MAX_ATTEMPTS - 1: + continue + raise last_error from exc - try: - value = schema.model_validate_json(choice.message.content or "") - except ValidationError as exc: - record.error_category = "invalid_output" self.ledger.record(record) - log.warning("model_output_invalid", extra={"attempt": attempt, "purpose": str(purpose)}) - last_error = ModelError("invalid_output", "model output failed schema validation") - if attempt < MAX_ATTEMPTS - 1: - continue - raise last_error from exc - - self.ledger.record(record) - log.info( - "model_request", - extra={ - "purpose": str(purpose), - "model": chosen.model, - "effort": chosen.reasoning_effort, - "ms": round(record.elapsed_ms), - "input": record.input_tokens, - "cached": record.cached_input_tokens, - "output": record.output_tokens, - "usd": round(record.estimated_usd, 6), - }, - ) - return StructuredResult(value, record) + log.info( + "model_request", + extra={ + "purpose": str(purpose), + "model": chosen.model, + "effort": chosen.reasoning_effort, + "ms": round(record.elapsed_ms), + "input": record.input_tokens, + "cached": record.cached_input_tokens, + "output": record.output_tokens, + "usd": round(record.estimated_usd, 6), + }, + ) + return StructuredResult(value, record) - assert last_error is not None - raise last_error + raise last_error or ModelError("invalid_output", "model request produced no result") diff --git a/services/api/quack/main.py b/services/api/quack/main.py index c901388..8c07a10 100644 --- a/services/api/quack/main.py +++ b/services/api/quack/main.py @@ -4,7 +4,6 @@ import asyncio import ipaddress -import json import logging import sys import time @@ -28,6 +27,7 @@ from quack.llm.postgres_ledger import PostgresLedger from quack.llm.report import summarise from quack.llm.router import ModelError, ModelRouter +from quack.observability import JsonFormatter, RequestDiagnostics from quack.schemas import ( AnalyzeRequest, AnalyzeResponse, @@ -63,7 +63,6 @@ from quack.services.voice import VoiceRejected, classify_intent, transcribe_wav from quack.services.writer_references import select_references -_STANDARD_LOG_FIELDS = set(logging.makeLogRecord({}).__dict__) | {"message", "asctime", "taskName"} MAX_REQUEST_BYTES = 10_000_000 MAX_AUDIO_BYTES = 8_000_000 MAX_TRACKED_CLIENTS = 10_000 @@ -73,13 +72,6 @@ EXPLAIN_DEADLINE_S = 25.0 -class JsonFormatter(logging.Formatter): - def format(self, record: logging.LogRecord) -> str: - payload = {"level": record.levelname, "logger": record.name, "event": record.getMessage()} - payload.update({k: v for k, v in record.__dict__.items() if k not in _STANDARD_LOG_FIELDS}) - return json.dumps(payload, default=str) - - def _configure_logging() -> None: handler = logging.StreamHandler(sys.stdout) handler.setFormatter(JsonFormatter()) @@ -148,7 +140,12 @@ def _same_origin(request: Request, allowed: list[str]) -> bool: return True # Host can be anything under DNS rebinding, where a site's name is pointed at this machine. So # beyond the configured origins, a match only counts on a local or private address. - parts = urlsplit(origin) + try: + parts = urlsplit(origin) + except ValueError: + return False + if parts.scheme not in ("http", "https") or parts.username or parts.password: + return False hosts = (request.headers.get("host"), request.headers.get("x-forwarded-host")) return _is_local_host(parts.hostname) and parts.netloc in hosts @@ -254,7 +251,7 @@ def _assistance( level: int, text: str, source: Literal["model", "template"], error: ModelError | None, started: float ) -> AssistanceResponse: total_ms = round((time.perf_counter() - started) * 1000, 1) - logging.getLogger("quack.help").info("help", extra={"level": level, "source": source, "total_ms": total_ms}) + logging.getLogger("quack.help").info("help", extra={"help_level": level, "source": source, "total_ms": total_ms}) return AssistanceResponse( level=level, text=text, source=source, model_issue=model_issue(error) if error else None, total_ms=total_ms ) @@ -285,6 +282,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: allow_origins=state.settings.cors_origin_list, allow_methods=["GET", "POST", "PUT", "DELETE"], allow_headers=["content-type", "authorization"], + expose_headers=["x-request-id"], ) app.add_middleware(BodySizeLimit, limit=MAX_REQUEST_BYTES) @@ -301,6 +299,8 @@ async def reject_cross_site(request: Request, call_next): # type: ignore[no-unt return JSONResponse({"detail": "cross-site request refused"}, status_code=403) return await call_next(request) + app.add_middleware(RequestDiagnostics) + def rate_limited(request: Request) -> None: client = request.client.host if request.client else "unknown" if not state.limiter.allow(client): diff --git a/services/api/quack/observability.py b/services/api/quack/observability.py new file mode 100644 index 0000000..b99c7a9 --- /dev/null +++ b/services/api/quack/observability.py @@ -0,0 +1,88 @@ +"""Request diagnostics with no bodies, credentials, user ids, or literal URL paths.""" + +from __future__ import annotations + +import json +import logging +import time +import traceback +from contextvars import ContextVar +from datetime import UTC, datetime +from pathlib import Path +from uuid import uuid4 + +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +request_id: ContextVar[str | None] = ContextVar("request_id", default=None) +log = logging.getLogger("quack.http") +_STANDARD_FIELDS = set(logging.makeLogRecord({}).__dict__) | {"message", "asctime", "taskName"} + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload = { + "timestamp": datetime.fromtimestamp(record.created, UTC).isoformat(), + "level": record.levelname, + "logger": record.name, + "event": record.getMessage(), + **{k: v for k, v in record.__dict__.items() if k not in _STANDARD_FIELDS}, + } + if current := request_id.get(): + payload["request_id"] = current + return json.dumps(payload, default=str, allow_nan=False) + + +class RequestDiagnostics: + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + # Always server-generated: a caller cannot inject log content or reuse another request's id. + correlation = uuid4().hex + token = request_id.set(correlation) + started = time.perf_counter() + status = 500 + response_started = False + + async def observed(message: Message) -> None: + nonlocal status, response_started + if message["type"] == "http.response.start": + status, response_started = message["status"], True + headers = list(message.get("headers", [])) + headers = [(k, v) for k, v in headers if k.lower() != b"x-request-id"] + headers.append((b"x-request-id", correlation.encode("ascii"))) + if scope.get("path", "").startswith("/api/"): + headers = [(k, v) for k, v in headers if k.lower() != b"cache-control"] + headers.append((b"cache-control", b"no-store")) + message = {**message, "headers": headers} + await send(message) + + try: + try: + await self.app(scope, receive, observed) + except Exception as exc: + # Exception messages can contain model text, SQL parameters or credentials. + frames = [ + {"file": Path(frame.f_code.co_filename).name, "line": line, "function": frame.f_code.co_name} + for frame, line in traceback.walk_tb(exc.__traceback__) + ][-8:] + log.error("request_failed", extra={"error_type": type(exc).__name__, "frames": frames}) + if response_started: + raise + await JSONResponse({"detail": "internal server error"}, status_code=500)(scope, receive, observed) + finally: + route = scope.get("route") + log.info( + "http_request", + extra={ + "method": scope["method"], + "route": getattr(route, "path", "unmatched"), + "status": status, + "duration_ms": round((time.perf_counter() - started) * 1000, 2), + }, + ) + request_id.reset(token) diff --git a/services/api/quack/schemas.py b/services/api/quack/schemas.py index df769af..0b659c2 100644 --- a/services/api/quack/schemas.py +++ b/services/api/quack/schemas.py @@ -2,9 +2,9 @@ from __future__ import annotations -from typing import Annotated, Any, Literal +from typing import Annotated, Any, Literal, Self -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from pydantic.alias_generators import to_camel AnalysisStatus = Literal["no_error", "confirmed_error", "uncertain", "unsupported"] @@ -210,6 +210,27 @@ class AnalyzeResponse(ApiModel): # Right as far as it goes, and with something asked for still to add: "missing_constant". reminder: str | None = None + @model_validator(mode="after") + def enforce_alert_evidence(self) -> Self: + """Refuse internally inconsistent alerts before they can reach a student's notebook. + + These checks run under python -O too; release safety cannot depend on assert statements. + The verifier remains the only source of the `verified` evidence itself. + """ + + ids = {mistake.id for mistake in self.mistakes} + if len(ids) != len(self.mistakes) or not all(mistake.verified for mistake in self.mistakes): + raise ValueError("mistakes must have unique ids and verified evidence") + if self.mistake is not None and (self.mistake not in self.mistakes or not self.mistake.verified): + raise ValueError("selected mistake must belong to the verified mistakes") + if ids.intersection(self.resolved_mistake_ids): + raise ValueError("a mistake cannot be present and resolved in the same response") + if self.intervention.should_quack and ( + self.status != "confirmed_error" or self.stale or not any(mistake.is_new for mistake in self.mistakes) + ): + raise ValueError("an alert needs a current, new, verified mistake") + return self + class ClearRequest(ApiModel): session_id: str = Field(min_length=1, max_length=64) diff --git a/services/api/quack/services/analysis.py b/services/api/quack/services/analysis.py index 5500a44..af166f4 100644 --- a/services/api/quack/services/analysis.py +++ b/services/api/quack/services/analysis.py @@ -1164,7 +1164,7 @@ def _lay_out( remembered = self._memory.get(session_id, ink.id) line = just_read if reread and just_read is not None else remembered or just_read if ( - overruled is not None and line is remembered and just_read is not None + overruled is not None and remembered is not None and line is remembered and just_read is not None and remembered.kind == just_read.kind == "math" and not _same_answer(remembered.math, just_read.math) ): overruled.add(ink.id) diff --git a/services/api/quack/services/handwriting.py b/services/api/quack/services/handwriting.py index ffe841e..a2e0538 100644 --- a/services/api/quack/services/handwriting.py +++ b/services/api/quack/services/handwriting.py @@ -6,7 +6,7 @@ import hashlib import io from dataclasses import dataclass -from typing import Literal +from typing import Any, Literal from PIL import Image, UnidentifiedImageError from pydantic import BaseModel, ConfigDict @@ -113,8 +113,9 @@ def sanitise_image(data_url: str) -> tuple[str, str]: if width * height > MAX_IMAGE_PIXELS: raise ImageRejected("image has too many pixels") probe.verify() - with Image.open(io.BytesIO(raw)) as image: - image.load() + with Image.open(io.BytesIO(raw)) as source: + source.load() + image: Image.Image = source if image.mode in ("RGBA", "LA", "P"): rgba = image.convert("RGBA") flattened = Image.new("RGB", rgba.size, "white") @@ -161,7 +162,7 @@ async def read_lines( content.append({"type": "text", "text": f"{lead} It has {line_count} written {noun}."}) content.append({"type": "image_url", "image_url": {"url": image}}) instructions = f"{COMPACT_PROMPT}\n\n{WRITER_REFERENCES_PROMPT}" if references else COMPACT_PROMPT - messages = [ + messages: list[dict[str, Any]] = [ {"role": "developer", "content": instructions}, {"role": "user", "content": content}, ] @@ -195,7 +196,7 @@ async def read_handwriting( } ) content.append({"type": "image_url", "image_url": {"url": page_context}}) - messages = [ + messages: list[dict[str, Any]] = [ # Stable instructions first, so that automatic prefix caching applies. {"role": "developer", "content": DEVELOPER_PROMPT}, {"role": "user", "content": content}, diff --git a/services/api/quack/services/hint_engine.py b/services/api/quack/services/hint_engine.py index 70d7174..33dcf05 100644 --- a/services/api/quack/services/hint_engine.py +++ b/services/api/quack/services/hint_engine.py @@ -972,4 +972,4 @@ async def get(self, session_id: str, mistake: TrackedMistake, level: Literal[1, def _log_failure(task: asyncio.Task[HelpResult]) -> None: if not task.cancelled() and task.exception() is not None: - log.warning("hint_preparation_failed", extra={"error": repr(task.exception())}) + log.warning("hint_preparation_failed", extra={"error_type": type(task.exception()).__name__}) diff --git a/services/api/quack/services/math_verifier.py b/services/api/quack/services/math_verifier.py index 3a12d5c..c44cbea 100644 --- a/services/api/quack/services/math_verifier.py +++ b/services/api/quack/services/math_verifier.py @@ -1226,7 +1226,7 @@ def _given(values: Values) -> dict[sp.Symbol, sp.Expr]: def _mentions_a_given(left: Node, right: Node, values: Values) -> bool: - if any(contains_kind(node, kind) for node in (left, right) for kind in ("deriv", "integral")): + if any(contains_kind(node, "deriv") or contains_kind(node, "integral") for node in (left, right)): return False letters = {letter for letter, _ in values} return bool(letters & (_written_variables(left) | _written_variables(right))) @@ -1826,9 +1826,10 @@ def _check_equation( if cur_values is not None: after_roots = cur_values else: - after_roots = _real_solutions(lhs_cur, rhs_cur, variable) - if after_roots is None: + solved_roots = _real_solutions(lhs_cur, rhs_cur, variable) + if solved_roots is None: return StepResult("unsupported", operation, line, from_line, reason="not_a_rational_equation") + after_roots = solved_roots same, after_within_before, before_within_after = _same_set(after_roots, before_roots) if same: @@ -1892,7 +1893,7 @@ def _check_equation( fingerprint=_fingerprint( "equation_solving", _canonical_equation(before), - _canonical_equation(after) if after is not None else sp.Tuple(*cur_values), + _canonical_equation(after) if after is not None else sp.Tuple(*(cur_values or [])), ), ) @@ -2128,14 +2129,15 @@ def _looks_like_identity_claim(left: Node, right: Node, line: LineInput, prior: except MathParseError: return False # `1 + 1 = 2` poses nothing to solve, so what comes after it isn't a step of solving it. - prior_equation = _is_equation(prior) and not _is_numeric_statement(prior) + prior_equation = prior is not None and _is_equation(prior) and not _is_numeric_statement(prior) if not a.free_symbols and not b.free_symbols: # After an equation, `1 = 2` is a result (no solutions) rather than a sum to check. if not prior_equation: return True - if _has_nothing_to_work_out(prior) and left.kind in ("add", "mul", "div", "pow") and _is_literal(right): + if (prior is not None and _has_nothing_to_work_out(prior) + and left.kind in ("add", "mul", "div", "pow") and _is_literal(right)): return True - return line.continues == "no" and not _could_follow_without_being_derived(prior, left, right) + return prior is not None and line.continues == "no" and not _could_follow_without_being_derived(prior, left, right) if (prior_equation and line.continues != "no") or line.operation not in SIMPLIFY_OPERATIONS: return False if sp.expand(a).free_symbols != b.free_symbols: @@ -2591,15 +2593,15 @@ def clears(context: _Context, stmt: Statement) -> bool: for prev in previous_readings: anchor = prev if ctx.anchor is ctx.prev else ctx.anchor context = replace(ctx, prev=prev, anchor=anchor) - for stmt in [original, *current_readings]: - if stmt is not None and clears(context, stmt): + for alternative in [original, *current_readings]: + if alternative is not None and clears(context, alternative): return True # An equation step can be compared with a line further back than the one above. if ctx.anchor_input is not None and ctx.anchor_input is not ctx.prev_line: for anchor in _parse_alternatives(ctx.anchor_input): context = replace(ctx, anchor=anchor) - for stmt in [original, *current_readings]: - if stmt is not None and clears(context, stmt): + for alternative in [original, *current_readings]: + if alternative is not None and clears(context, alternative): return True return False diff --git a/services/api/quack/services/practice.py b/services/api/quack/services/practice.py index a74059c..58d878e 100644 --- a/services/api/quack/services/practice.py +++ b/services/api/quack/services/practice.py @@ -14,13 +14,14 @@ from dataclasses import dataclass from fractions import Fraction +from quack.schemas import PracticeTopic from quack.services.handwriting import RESTATED_QUESTION, HandwritingReading, ReadLine from quack.services.math_parser import MathParseError, normalise, parse_expression, parse_statement from quack.services.math_verifier import LineInput, TaskIntent, names_given_values from quack.services.verifier_pool import VerificationFailed, shared_pool MAX_TRIES = 8 -SUPPORTED = ( +SUPPORTED: tuple[PracticeTopic, ...] = ( "addition_subtraction", "multiplication", "division", diff --git a/services/api/quack/services/starter_questions.py b/services/api/quack/services/starter_questions.py index 2582bc3..dbdba6a 100644 --- a/services/api/quack/services/starter_questions.py +++ b/services/api/quack/services/starter_questions.py @@ -17,6 +17,7 @@ import random from fractions import Fraction +from functools import partial from typing import Any from quack.services.lesson_questions import ( @@ -29,6 +30,7 @@ TOPIC_OF_SKILL, LessonQuestion, Template, + Tier, _lesson, _pick, ) @@ -245,7 +247,7 @@ def _proper(rng: random.Random, fixed: dict[str, Any] | None, largest: int) -> t return top, bottom -def _simplify(template: str, tier: str, factors: tuple[int, int]): # type: ignore[no-untyped-def] +def _simplify(template: str, tier: Tier, factors: tuple[int, int]): # type: ignore[no-untyped-def] def build(rng: random.Random, fixed: dict[str, Any] | None) -> LessonQuestion: top, bottom = (fixed["p"], fixed["q"]) if fixed and "p" in fixed else _proper(rng, fixed, 7) k = _pick(rng, fixed, "k", lambda: rng.randint(*factors)) @@ -340,7 +342,7 @@ def _subtract_fractions(rng: random.Random, fixed: dict[str, Any] | None) -> Les def _to_decimal(rng: random.Random, fixed: dict[str, Any] | None) -> LessonQuestion: while True: bottom = _pick(rng, fixed, "q", lambda: rng.choice(_DECIMAL_BOTTOMS)) - top = _pick(rng, fixed, "p", lambda bottom=bottom: rng.randint(1, bottom - 1)) + top = _pick(rng, fixed, "p", partial(rng.randint, 1, bottom - 1)) if Fraction(top, bottom).denominator == bottom: break fixed = None @@ -361,7 +363,7 @@ def _to_decimal(rng: random.Random, fixed: dict[str, Any] | None) -> LessonQuest def _to_fraction(rng: random.Random, fixed: dict[str, Any] | None) -> LessonQuestion: while True: bottom = _pick(rng, fixed, "q", lambda: rng.choice((4, 5, 20, 25))) - top = _pick(rng, fixed, "p", lambda bottom=bottom: rng.randint(1, bottom - 1)) + top = _pick(rng, fixed, "p", partial(rng.randint, 1, bottom - 1)) if Fraction(top, bottom).denominator == bottom: break fixed = None @@ -518,7 +520,7 @@ def _odd_triangle_area(rng: random.Random, fixed: dict[str, Any] | None) -> Less _RATIO_SIDES = {"sin": ("opposite", "hypotenuse"), "cos": ("adjacent", "hypotenuse"), "tan": ("opposite", "adjacent")} -def _ratio(template: str, tier: str, ratios: tuple[str, ...]): # type: ignore[no-untyped-def] +def _ratio(template: str, tier: Tier, ratios: tuple[str, ...]): # type: ignore[no-untyped-def] def build(rng: random.Random, fixed: dict[str, Any] | None) -> LessonQuestion: opposite, adjacent, hypotenuse = _pick(rng, fixed, "triple", lambda: rng.choice(_TRIPLES)) ratio = _pick(rng, fixed, "ratio", lambda: rng.choice(ratios)) diff --git a/services/api/quack/services/verifier_pool.py b/services/api/quack/services/verifier_pool.py index df5bfe1..f584d27 100644 --- a/services/api/quack/services/verifier_pool.py +++ b/services/api/quack/services/verifier_pool.py @@ -69,7 +69,8 @@ def _child(conn: Connection, job: object, *args: object) -> None: try: conn.send(("ok", job(*args))) # type: ignore[operator] except BaseException as exc: # noqa: BLE001 - conn.send(("error", repr(exc))) + # Exception messages can include the student's parsed expression. + conn.send(("error", type(exc).__name__)) finally: conn.close() @@ -118,7 +119,7 @@ def _run_job(self, job: object, args: tuple[object, ...], timeout_s: float): # process.join(1) if status != "ok": # A bug in checking one line shouldn't take down analysis. Quack just stays quiet. - log.error("verification_error", extra={"error": payload}) + log.error("verification_error", extra={"error_type": payload}) raise VerificationFailed("error") return payload diff --git a/services/api/requirements-dev.txt b/services/api/requirements-dev.txt index bedd186..96b2efa 100644 --- a/services/api/requirements-dev.txt +++ b/services/api/requirements-dev.txt @@ -4,3 +4,4 @@ pytest-asyncio>=0.24,<2 respx>=0.21,<1 ruff>=0.6,<1 mypy>=1.11,<2 +-r requirements-deploy.txt diff --git a/services/api/requirements-tooling.lock b/services/api/requirements-tooling.lock new file mode 100644 index 0000000..0269773 --- /dev/null +++ b/services/api/requirements-tooling.lock @@ -0,0 +1,20 @@ +# Pinned deployment-test tooling. Install with requirements.lock. +aiohappyeyeballs==2.7.1 +aiohttp==3.14.5 +aiosignal==1.4.0 +attrs==26.1.0 +cbor2==6.2.1 +frozenlist==1.8.0 +grpclib==0.4.9 +h2==4.4.1 +hpack==4.2.0 +hyperframe==6.1.0 +modal==1.6.3 +multidict==7.2.1 +propcache==0.5.4 +protobuf==7.36.2 +synchronicity==0.12.7 +toml==0.10.2 +types-certifi==2021.10.8.3 +types-toml==0.10.8.20260518 +yarl==1.25.1 diff --git a/services/api/tests/accounts_support.py b/services/api/tests/accounts_support.py index 141e72e..0291070 100644 --- a/services/api/tests/accounts_support.py +++ b/services/api/tests/accounts_support.py @@ -40,6 +40,9 @@ def _empty_postgres() -> None: return import psycopg + info = psycopg.conninfo.conninfo_to_dict(postgres) + if info.get("dbname") != "quack_accounts_test" or info.get("host") not in {"127.0.0.1", "localhost", "::1"}: + pytest.fail("account tests require a local disposable database named quack_accounts_test") with psycopg.connect(postgres, autocommit=True) as connection: # Each test's server leaves a connection open, and they'd soon use up the limit. connection.execute("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid()") diff --git a/services/api/tests/conftest.py b/services/api/tests/conftest.py index 2b15a20..2e327cf 100644 --- a/services/api/tests/conftest.py +++ b/services/api/tests/conftest.py @@ -1,3 +1,4 @@ +import os from pathlib import Path import pytest @@ -6,6 +7,23 @@ from tests.accounts_support import _empty_postgres, accounts_client # noqa: F401 +def pytest_sessionfinish(session: pytest.Session) -> None: + if os.environ.get("QUACK_RELEASE_VERIFY") == "1": + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter and reporter.stats.get("skipped"): + reporter.write_sep("=", "Release verification refuses skipped tests") + session.exitstatus = pytest.ExitCode.TESTS_FAILED + + +@pytest.fixture(autouse=True) +def isolate_mocked_http(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: + """Mocked HTTP must not depend on a developer's or CI runner's proxy setup.""" + + if request.node.get_closest_marker("live") is None: + for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + monkeypatch.delenv(name, raising=False) + + @pytest.fixture def settings(tmp_path: Path) -> Settings: return Settings( diff --git a/services/api/tests/fixtures/recorded-routing.json b/services/api/tests/fixtures/recorded-routing.json new file mode 100644 index 0000000..dc94418 --- /dev/null +++ b/services/api/tests/fixtures/recorded-routing.json @@ -0,0 +1,371 @@ +{ + "description": "Historical classifications on the published synthetic stroke dataset. Only case/config/status and hashed fingerprint equality are retained; no model text, usage, paths or student records. These are historical prompt regressions, not a current accuracy benchmark.", + "runs": { + "v5-all": [ + {"case":"st_v_mul_3x4","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_mul_2x3","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_v_mul_2x3","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_mul_3x4","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_i_bigx_2x3","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_i_bigx_3x4","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_v_bigx_3x4","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_bigx_2x3","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_add","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"eb06db0df854904167205a7daf662e7fd4aa3320012f0c59031342c71773a770"}, + {"case":"st_v_add","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_div","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_div","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c552db87e0f43b07e518c459317f3eb5306b5890b1f27f464d0cf184650765d1"}, + {"case":"st_i_diff_power","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"27fc1e0276480da656072b5157049c00d2a5279a4d8c9a3f3a594d18e4ed8c80"}, + {"case":"st_i_dist","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"st_i_diff","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"st_v_diff","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"demo_algebra","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_correct","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_expand","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_two_brackets","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"demo_diff","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"alg_v_negative","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_like_terms","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_square","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_factor","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_arith","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_chain","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_unconventional","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_equation","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_distribution","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e1141c11f1d72004b667721d625457f4bedf70061aee83121e30ab073bcee9bf"}, + {"case":"alg_i_square","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"5a2ebe1cd6298981aace9f4d01b84eb55ef82f379dbf737829f03befe791efd1"}, + {"case":"alg_v_quadratic","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_arith","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"58442325e681a7eb66ad433444459dd70d751a40d240eba3af1eb90321dcc642"}, + {"case":"alg_i_like_terms","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"0981a9217075da65f72d8244a5c881469ede06e30446538f37a2b630e83a05d8"}, + {"case":"alg_i_sign","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"b3b93fb297a94023701b9c9ec6bc8655b354e5013c3699eab331ce8b135e00c3"}, + {"case":"alg_i_two_brackets","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"3a3fac44294f97ac1ee410a45acf9b102d238561f408ee18ddb439bdfad10bfa"}, + {"case":"alg_i_equation_divide","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"824ce6da1f0844ac7f323307b23909dc3c26d1eb901128347bd6aaa76e27be72"}, + {"case":"alg_i_factor","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c735cb26eea690d7a3c52fae89fc558c3faa4ed7fdc10cb4e5c93360c3d8fd89"}, + {"case":"alg_i_equation_sign","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"18d193684972a86c8ddd857fff19d2182e3b3a3fddcb06472f44bca51e3d56b8"}, + {"case":"alg_i_later_step","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"90e3f414eb263119efa29fffd90be12fe63b281b4c9935e8ed0eda4abaecb114"}, + {"case":"diff_v_constant","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_power","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sum","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_poly","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_leibniz","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_function","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sin","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_i_exponent","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e6ffbc72abc45894e8b6f38505d7f90486f297d9ba39717be4e422acd8929116"}, + {"case":"diff_v_chain","config":"A","expected":"no_error","predicted":"uncertain","fingerprint":null}, + {"case":"diff_i_constant","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"4dcfc8cec5ae4ccc1f6faec2bb3352dd946d8e6f76d1847bf7bdd201e4881f9e"}, + {"case":"diff_i_coefficient","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"7ef1a56bbbe0fbe7adf87bd8ef561f8e7d06dda562b5a31977178227309a8e4f"}, + {"case":"diff_i_missing","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"befe77023a414fae34dadba3da42915142362779cd01d5d274102d7a8fc1e833"}, + {"case":"diff_i_function","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"24cc3dea012470a6f45e7539700f9a0d8cea6e0d2ab94b25c38aec3e984c9320"}, + {"case":"diff_i_leibniz","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a095ec1e895734cdb3ab5ec498bc4a7ab1f69449462c55ce92b77f9994cacfeb"}, + {"case":"diff_i_sin","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"6910e10d635fa5a411494361dd0eef03073af077a0c7f1ead135569ec10d433b"}, + {"case":"diff_i_chain","config":"A","expected":"confirmed_error","predicted":"uncertain","fingerprint":null}, + {"case":"abs_incomplete","config":"A","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_integral","config":"A","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_inequality","config":"A","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_substitution","config":"A","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_lost_root","config":"A","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_messy_invalid","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"ac8bfef7ed8c7604698f84532321cc417367a6f39c5d9c449df1aa8b1fc37d30"}, + {"case":"abs_domain_sqrt","config":"A","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_overlap_invalid","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"47d7aa89f19bf83ad6b1a1dc70dc553c1379fcb0c82797948cdb344d0ba7c61f"}, + {"case":"hard_overlap_valid","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_messy_valid","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_valid","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_fraction_valid","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_fraction_invalid","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"d7c9aab23af77e60f44face279dacc6d8fb39fe4e1307314c1ae3563e67b16bf"}, + {"case":"hard_super_invalid","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"1e1488af4943af3c3ad28c02fe8ee23b5e741eef72bc1355f31135df6499f994"}, + {"case":"hard_crossed_out","config":"A","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_two_errors","config":"A","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"} + ], + "v6-compact": [ + {"case":"st_i_mul_2x3","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_v_mul_2x3","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_mul_3x4","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_mul_3x4","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_i_bigx_2x3","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_v_bigx_2x3","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_bigx_3x4","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_div","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c552db87e0f43b07e518c459317f3eb5306b5890b1f27f464d0cf184650765d1"}, + {"case":"st_i_bigx_3x4","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_v_div","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_add","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"eb06db0df854904167205a7daf662e7fd4aa3320012f0c59031342c71773a770"}, + {"case":"st_v_add","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_diff","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"st_v_diff","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_diff_power","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"27fc1e0276480da656072b5157049c00d2a5279a4d8c9a3f3a594d18e4ed8c80"}, + {"case":"st_i_dist","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_diff","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"demo_algebra","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_correct","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_expand","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_two_brackets","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_negative","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_like_terms","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_square","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_factor","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_chain","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_arith","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_equation","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_unconventional","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_distribution","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e1141c11f1d72004b667721d625457f4bedf70061aee83121e30ab073bcee9bf"}, + {"case":"alg_v_quadratic","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_square","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"5a2ebe1cd6298981aace9f4d01b84eb55ef82f379dbf737829f03befe791efd1"}, + {"case":"alg_i_arith","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"58442325e681a7eb66ad433444459dd70d751a40d240eba3af1eb90321dcc642"}, + {"case":"alg_i_sign","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"b3b93fb297a94023701b9c9ec6bc8655b354e5013c3699eab331ce8b135e00c3"}, + {"case":"alg_i_two_brackets","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"3a3fac44294f97ac1ee410a45acf9b102d238561f408ee18ddb439bdfad10bfa"}, + {"case":"alg_i_like_terms","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"0981a9217075da65f72d8244a5c881469ede06e30446538f37a2b630e83a05d8"}, + {"case":"alg_i_factor","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c735cb26eea690d7a3c52fae89fc558c3faa4ed7fdc10cb4e5c93360c3d8fd89"}, + {"case":"alg_i_equation_sign","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"18d193684972a86c8ddd857fff19d2182e3b3a3fddcb06472f44bca51e3d56b8"}, + {"case":"alg_i_equation_divide","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"824ce6da1f0844ac7f323307b23909dc3c26d1eb901128347bd6aaa76e27be72"}, + {"case":"alg_i_later_step","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"90e3f414eb263119efa29fffd90be12fe63b281b4c9935e8ed0eda4abaecb114"}, + {"case":"diff_v_sum","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_power","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_constant","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_poly","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_leibniz","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_function","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sin","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_i_exponent","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e6ffbc72abc45894e8b6f38505d7f90486f297d9ba39717be4e422acd8929116"}, + {"case":"diff_v_chain","config":"H","expected":"no_error","predicted":"uncertain","fingerprint":null}, + {"case":"diff_i_coefficient","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"7ef1a56bbbe0fbe7adf87bd8ef561f8e7d06dda562b5a31977178227309a8e4f"}, + {"case":"diff_i_constant","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"4dcfc8cec5ae4ccc1f6faec2bb3352dd946d8e6f76d1847bf7bdd201e4881f9e"}, + {"case":"diff_i_missing","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"befe77023a414fae34dadba3da42915142362779cd01d5d274102d7a8fc1e833"}, + {"case":"diff_i_leibniz","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a095ec1e895734cdb3ab5ec498bc4a7ab1f69449462c55ce92b77f9994cacfeb"}, + {"case":"diff_i_function","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"24cc3dea012470a6f45e7539700f9a0d8cea6e0d2ab94b25c38aec3e984c9320"}, + {"case":"diff_i_sin","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"6910e10d635fa5a411494361dd0eef03073af077a0c7f1ead135569ec10d433b"}, + {"case":"abs_integral","config":"H","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_inequality","config":"H","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_substitution","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"diff_i_chain","config":"H","expected":"confirmed_error","predicted":"uncertain","fingerprint":null}, + {"case":"abs_incomplete","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_domain_sqrt","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_messy_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"ac8bfef7ed8c7604698f84532321cc417367a6f39c5d9c449df1aa8b1fc37d30"}, + {"case":"abs_lost_root","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_overlap_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"47d7aa89f19bf83ad6b1a1dc70dc553c1379fcb0c82797948cdb344d0ba7c61f"}, + {"case":"hard_messy_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_overlap_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_fraction_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"1e1488af4943af3c3ad28c02fe8ee23b5e741eef72bc1355f31135df6499f994"}, + {"case":"st_i_mul_2x3","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"hard_fraction_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"d7c9aab23af77e60f44face279dacc6d8fb39fe4e1307314c1ae3563e67b16bf"}, + {"case":"st_i_mul_3x4","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_v_mul_2x3","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_mul_3x4","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_two_errors","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"st_i_bigx_2x3","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_v_bigx_2x3","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_bigx_3x4","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_v_div","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_bigx_3x4","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_div","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c552db87e0f43b07e518c459317f3eb5306b5890b1f27f464d0cf184650765d1"}, + {"case":"st_i_add","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"eb06db0df854904167205a7daf662e7fd4aa3320012f0c59031342c71773a770"}, + {"case":"st_v_add","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_diff","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"st_i_diff_power","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"27fc1e0276480da656072b5157049c00d2a5279a4d8c9a3f3a594d18e4ed8c80"}, + {"case":"st_v_diff","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_dist","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_algebra","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_diff","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"demo_correct","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_expand","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_two_brackets","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_negative","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_like_terms","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_crossed_out","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_square","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_arith","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_factor","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_chain","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_equation","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_unconventional","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_quadratic","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_distribution","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e1141c11f1d72004b667721d625457f4bedf70061aee83121e30ab073bcee9bf"}, + {"case":"alg_i_square","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"5a2ebe1cd6298981aace9f4d01b84eb55ef82f379dbf737829f03befe791efd1"}, + {"case":"alg_i_sign","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"b3b93fb297a94023701b9c9ec6bc8655b354e5013c3699eab331ce8b135e00c3"}, + {"case":"alg_i_arith","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"58442325e681a7eb66ad433444459dd70d751a40d240eba3af1eb90321dcc642"}, + {"case":"alg_i_like_terms","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"0981a9217075da65f72d8244a5c881469ede06e30446538f37a2b630e83a05d8"}, + {"case":"alg_i_two_brackets","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"3a3fac44294f97ac1ee410a45acf9b102d238561f408ee18ddb439bdfad10bfa"}, + {"case":"alg_i_factor","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c735cb26eea690d7a3c52fae89fc558c3faa4ed7fdc10cb4e5c93360c3d8fd89"}, + {"case":"alg_i_equation_sign","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"18d193684972a86c8ddd857fff19d2182e3b3a3fddcb06472f44bca51e3d56b8"}, + {"case":"alg_i_equation_divide","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"824ce6da1f0844ac7f323307b23909dc3c26d1eb901128347bd6aaa76e27be72"}, + {"case":"diff_v_power","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_later_step","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"90e3f414eb263119efa29fffd90be12fe63b281b4c9935e8ed0eda4abaecb114"}, + {"case":"diff_v_sum","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_constant","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_leibniz","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_poly","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_function","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sin","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_i_coefficient","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"7ef1a56bbbe0fbe7adf87bd8ef561f8e7d06dda562b5a31977178227309a8e4f"}, + {"case":"diff_i_exponent","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e6ffbc72abc45894e8b6f38505d7f90486f297d9ba39717be4e422acd8929116"}, + {"case":"diff_v_chain","config":"I","expected":"no_error","predicted":"uncertain","fingerprint":null}, + {"case":"diff_i_leibniz","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a095ec1e895734cdb3ab5ec498bc4a7ab1f69449462c55ce92b77f9994cacfeb"}, + {"case":"diff_i_constant","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"4dcfc8cec5ae4ccc1f6faec2bb3352dd946d8e6f76d1847bf7bdd201e4881f9e"}, + {"case":"diff_i_missing","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"befe77023a414fae34dadba3da42915142362779cd01d5d274102d7a8fc1e833"}, + {"case":"diff_i_function","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"24cc3dea012470a6f45e7539700f9a0d8cea6e0d2ab94b25c38aec3e984c9320"}, + {"case":"diff_i_sin","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"6910e10d635fa5a411494361dd0eef03073af077a0c7f1ead135569ec10d433b"}, + {"case":"abs_integral","config":"I","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_inequality","config":"I","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"diff_i_chain","config":"I","expected":"confirmed_error","predicted":"uncertain","fingerprint":null}, + {"case":"abs_incomplete","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_domain_sqrt","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_substitution","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_lost_root","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_messy_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"ac8bfef7ed8c7604698f84532321cc417367a6f39c5d9c449df1aa8b1fc37d30"}, + {"case":"hard_overlap_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"47d7aa89f19bf83ad6b1a1dc70dc553c1379fcb0c82797948cdb344d0ba7c61f"}, + {"case":"hard_overlap_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_messy_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"1e1488af4943af3c3ad28c02fe8ee23b5e741eef72bc1355f31135df6499f994"}, + {"case":"hard_fraction_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_fraction_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"d7c9aab23af77e60f44face279dacc6d8fb39fe4e1307314c1ae3563e67b16bf"}, + {"case":"hard_two_errors","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"hard_crossed_out","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null} + ], + "v7-compact": [ + {"case":"st_v_mul_2x3","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_mul_3x4","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_mul_2x3","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_i_mul_3x4","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_i_bigx_3x4","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_v_bigx_2x3","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_bigx_2x3","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_v_bigx_3x4","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_div","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c552db87e0f43b07e518c459317f3eb5306b5890b1f27f464d0cf184650765d1"}, + {"case":"st_i_add","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"eb06db0df854904167205a7daf662e7fd4aa3320012f0c59031342c71773a770"}, + {"case":"st_v_add","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_div","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_diff","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"st_i_diff_power","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"27fc1e0276480da656072b5157049c00d2a5279a4d8c9a3f3a594d18e4ed8c80"}, + {"case":"st_v_diff","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_dist","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_algebra","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_correct","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"demo_diff","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"alg_v_expand","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_two_brackets","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_negative","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_square","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_like_terms","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_arith","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_chain","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_factor","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_unconventional","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_equation","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_quadratic","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_distribution","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e1141c11f1d72004b667721d625457f4bedf70061aee83121e30ab073bcee9bf"}, + {"case":"alg_i_square","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"5a2ebe1cd6298981aace9f4d01b84eb55ef82f379dbf737829f03befe791efd1"}, + {"case":"alg_i_arith","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"58442325e681a7eb66ad433444459dd70d751a40d240eba3af1eb90321dcc642"}, + {"case":"alg_i_like_terms","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"0981a9217075da65f72d8244a5c881469ede06e30446538f37a2b630e83a05d8"}, + {"case":"alg_i_sign","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"b3b93fb297a94023701b9c9ec6bc8655b354e5013c3699eab331ce8b135e00c3"}, + {"case":"alg_i_two_brackets","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"3a3fac44294f97ac1ee410a45acf9b102d238561f408ee18ddb439bdfad10bfa"}, + {"case":"alg_i_factor","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c735cb26eea690d7a3c52fae89fc558c3faa4ed7fdc10cb4e5c93360c3d8fd89"}, + {"case":"alg_i_equation_divide","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"824ce6da1f0844ac7f323307b23909dc3c26d1eb901128347bd6aaa76e27be72"}, + {"case":"alg_i_equation_sign","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"18d193684972a86c8ddd857fff19d2182e3b3a3fddcb06472f44bca51e3d56b8"}, + {"case":"diff_v_power","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sum","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_later_step","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"90e3f414eb263119efa29fffd90be12fe63b281b4c9935e8ed0eda4abaecb114"}, + {"case":"diff_v_poly","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_constant","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_leibniz","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_function","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sin","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_i_exponent","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e6ffbc72abc45894e8b6f38505d7f90486f297d9ba39717be4e422acd8929116"}, + {"case":"diff_i_coefficient","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"7ef1a56bbbe0fbe7adf87bd8ef561f8e7d06dda562b5a31977178227309a8e4f"}, + {"case":"diff_v_chain","config":"H","expected":"no_error","predicted":"uncertain","fingerprint":null}, + {"case":"diff_i_constant","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"4dcfc8cec5ae4ccc1f6faec2bb3352dd946d8e6f76d1847bf7bdd201e4881f9e"}, + {"case":"diff_i_function","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"24cc3dea012470a6f45e7539700f9a0d8cea6e0d2ab94b25c38aec3e984c9320"}, + {"case":"diff_i_leibniz","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a095ec1e895734cdb3ab5ec498bc4a7ab1f69449462c55ce92b77f9994cacfeb"}, + {"case":"diff_i_missing","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"befe77023a414fae34dadba3da42915142362779cd01d5d274102d7a8fc1e833"}, + {"case":"diff_i_chain","config":"H","expected":"confirmed_error","predicted":"uncertain","fingerprint":null}, + {"case":"abs_inequality","config":"H","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"diff_i_sin","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"6910e10d635fa5a411494361dd0eef03073af077a0c7f1ead135569ec10d433b"}, + {"case":"abs_integral","config":"H","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_incomplete","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_domain_sqrt","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_substitution","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_lost_root","config":"H","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_overlap_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"47d7aa89f19bf83ad6b1a1dc70dc553c1379fcb0c82797948cdb344d0ba7c61f"}, + {"case":"hard_messy_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"ac8bfef7ed8c7604698f84532321cc417367a6f39c5d9c449df1aa8b1fc37d30"}, + {"case":"hard_overlap_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"1e1488af4943af3c3ad28c02fe8ee23b5e741eef72bc1355f31135df6499f994"}, + {"case":"hard_fraction_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_messy_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_super_valid","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_mul_2x3","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"hard_two_errors","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"hard_fraction_invalid","config":"H","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"d7c9aab23af77e60f44face279dacc6d8fb39fe4e1307314c1ae3563e67b16bf"}, + {"case":"st_v_mul_2x3","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_mul_3x4","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_v_mul_3x4","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_bigx_3x4","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8de3a073bfe6bc108ec95297f6cbd0a7481f6769d146eb450ec32d59a0c32c56"}, + {"case":"st_i_bigx_2x3","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"8093854207bcd002d18727b5ddd6d5bb35ce91b7d8ac06c818ee8f6e7ac71e31"}, + {"case":"st_v_bigx_2x3","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_bigx_3x4","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_div","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c552db87e0f43b07e518c459317f3eb5306b5890b1f27f464d0cf184650765d1"}, + {"case":"st_v_div","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_v_add","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_add","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"eb06db0df854904167205a7daf662e7fd4aa3320012f0c59031342c71773a770"}, + {"case":"st_i_diff","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"st_v_diff","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"st_i_diff_power","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"27fc1e0276480da656072b5157049c00d2a5279a4d8c9a3f3a594d18e4ed8c80"}, + {"case":"st_i_dist","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_algebra","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"demo_correct","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"demo_diff","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"f51a477905281b99340310f6929238752814f2aecbb96169691832b1b9912ce0"}, + {"case":"alg_v_expand","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_like_terms","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_negative","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_two_brackets","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_square","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_arith","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_factor","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_crossed_out","config":"H","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_chain","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_equation","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_unconventional","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_v_quadratic","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"alg_i_distribution","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e1141c11f1d72004b667721d625457f4bedf70061aee83121e30ab073bcee9bf"}, + {"case":"alg_i_sign","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"b3b93fb297a94023701b9c9ec6bc8655b354e5013c3699eab331ce8b135e00c3"}, + {"case":"alg_i_square","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"5a2ebe1cd6298981aace9f4d01b84eb55ef82f379dbf737829f03befe791efd1"}, + {"case":"alg_i_arith","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"58442325e681a7eb66ad433444459dd70d751a40d240eba3af1eb90321dcc642"}, + {"case":"alg_i_two_brackets","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"3a3fac44294f97ac1ee410a45acf9b102d238561f408ee18ddb439bdfad10bfa"}, + {"case":"alg_i_like_terms","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"0981a9217075da65f72d8244a5c881469ede06e30446538f37a2b630e83a05d8"}, + {"case":"alg_i_factor","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"c735cb26eea690d7a3c52fae89fc558c3faa4ed7fdc10cb4e5c93360c3d8fd89"}, + {"case":"alg_i_equation_sign","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"18d193684972a86c8ddd857fff19d2182e3b3a3fddcb06472f44bca51e3d56b8"}, + {"case":"alg_i_later_step","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"90e3f414eb263119efa29fffd90be12fe63b281b4c9935e8ed0eda4abaecb114"}, + {"case":"alg_i_equation_divide","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"824ce6da1f0844ac7f323307b23909dc3c26d1eb901128347bd6aaa76e27be72"}, + {"case":"diff_v_sum","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_power","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_constant","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_poly","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_leibniz","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_function","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_sin","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_v_chain","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"diff_i_exponent","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"e6ffbc72abc45894e8b6f38505d7f90486f297d9ba39717be4e422acd8929116"}, + {"case":"diff_i_coefficient","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"7ef1a56bbbe0fbe7adf87bd8ef561f8e7d06dda562b5a31977178227309a8e4f"}, + {"case":"diff_i_constant","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"4dcfc8cec5ae4ccc1f6faec2bb3352dd946d8e6f76d1847bf7bdd201e4881f9e"}, + {"case":"diff_i_missing","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"befe77023a414fae34dadba3da42915142362779cd01d5d274102d7a8fc1e833"}, + {"case":"diff_i_leibniz","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a095ec1e895734cdb3ab5ec498bc4a7ab1f69449462c55ce92b77f9994cacfeb"}, + {"case":"diff_i_function","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"24cc3dea012470a6f45e7539700f9a0d8cea6e0d2ab94b25c38aec3e984c9320"}, + {"case":"diff_i_sin","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"6910e10d635fa5a411494361dd0eef03073af077a0c7f1ead135569ec10d433b"}, + {"case":"abs_inequality","config":"I","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"diff_i_chain","config":"I","expected":"confirmed_error","predicted":"uncertain","fingerprint":null}, + {"case":"abs_integral","config":"I","expected":"abstain","predicted":"unsupported","fingerprint":null}, + {"case":"abs_incomplete","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_lost_root","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_substitution","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"abs_domain_sqrt","config":"I","expected":"abstain","predicted":"uncertain","fingerprint":null}, + {"case":"hard_messy_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"ac8bfef7ed8c7604698f84532321cc417367a6f39c5d9c449df1aa8b1fc37d30"}, + {"case":"hard_messy_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_overlap_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"47d7aa89f19bf83ad6b1a1dc70dc553c1379fcb0c82797948cdb344d0ba7c61f"}, + {"case":"hard_super_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_fraction_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_overlap_valid","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null}, + {"case":"hard_fraction_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"d7c9aab23af77e60f44face279dacc6d8fb39fe4e1307314c1ae3563e67b16bf"}, + {"case":"hard_super_invalid","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"1e1488af4943af3c3ad28c02fe8ee23b5e741eef72bc1355f31135df6499f994"}, + {"case":"hard_two_errors","config":"I","expected":"confirmed_error","predicted":"confirmed_error","fingerprint":"a012fcaac92278d1872b193c0f95c2b02a69ef58b2326aa53f72e26875949c4f"}, + {"case":"hard_crossed_out","config":"I","expected":"no_error","predicted":"no_error","fingerprint":null} + ] + } +} diff --git a/services/api/tests/test_postgres_ledger.py b/services/api/tests/test_postgres_ledger.py index 475d6ba..97b03a5 100644 --- a/services/api/tests/test_postgres_ledger.py +++ b/services/api/tests/test_postgres_ledger.py @@ -1,14 +1,20 @@ """Run only against a disposable database named quack_ledger_test.""" +import asyncio import os from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from dataclasses import replace +import httpx import psycopg import pytest +import respx from quack.llm.ledger import BudgetExceeded, UsageRecord from quack.llm.postgres_ledger import PostgresLedger +from quack.llm.router import DEFAULT_ROUTES, MAX_ATTEMPTS, ModelError, ModelRouter, Purpose +from tests.test_release_safety import Answer, completion @pytest.fixture @@ -16,8 +22,9 @@ def url(): # type: ignore[no-untyped-def] value = os.environ.get("QUACK_LEDGER_TEST_DATABASE_URL", "") if not value: pytest.skip("set QUACK_LEDGER_TEST_DATABASE_URL to a disposable quack_ledger_test database") - if psycopg.conninfo.conninfo_to_dict(value).get("dbname") != "quack_ledger_test": - pytest.fail("refusing ledger tests against a database not named quack_ledger_test") + info = psycopg.conninfo.conninfo_to_dict(value) + if info.get("dbname") != "quack_ledger_test" or info.get("host") not in {"127.0.0.1", "localhost", "::1"}: + pytest.fail("ledger tests require a local disposable database named quack_ledger_test") ledger = PostgresLedger(value, 1, 1, 1, 1) ledger.database.execute("TRUNCATE quack_model_usage, quack_model_debits") ledger.database.close() @@ -130,3 +137,73 @@ def test_quota_and_usage_tables_have_no_public_rls_access(url): # type: ignore[ connection.execute("SET ROLE quack_ledger_public") for table in ("quack_model_debits", "quack_model_usage"): assert connection.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] == 0 + + +@respx.mock +@pytest.mark.parametrize("failure", ["timeout", "invalid_output"]) +async def test_router_retries_share_one_durable_operation_debit(url, settings, monkeypatch, failure): # type: ignore[no-untyped-def] + monkeypatch.setattr("quack.llm.router._retry_delay", lambda *_: 0) + first = httpx.ReadTimeout("slow") if failure == "timeout" else completion("not JSON") + route = respx.post("https://api.test/v1/chat/completions").mock(side_effect=[first, completion()]) + with ledger(url) as instance: + router = ModelRouter(settings, instance) + result = await router.complete_json( + Purpose.EXPLAIN, [], Answer, session_id="alice:s", + route=replace(DEFAULT_ROUTES[Purpose.EXPLAIN], retry_timeouts=True), + ) + assert result.value.answer == "ok" + assert route.call_count == 2 + debits = instance.database.query("SELECT * FROM quack_model_debits") + assert len(debits) == 1 + receipts = instance.database.query("SELECT * FROM quack_model_usage") + assert len(receipts) == 2 + assert {row["debit_id"] for row in receipts} == {debits[0]["id"]} + assert [row["record"]["attempt"] for row in receipts] == [1, 2] + # Missing/failed usage cannot refund any part of the original retry allowance. + assert instance.project_spend > sum(float(row["estimated_usd"]) for row in receipts) + + +@respx.mock +async def test_router_cancellation_records_receipt_and_retains_reserve_after_restart(url, settings): # type: ignore[no-untyped-def] + called = asyncio.Event() + + async def waiting(_): # type: ignore[no-untyped-def] + called.set() + await asyncio.Event().wait() + return completion() + + respx.post("https://api.test/v1/chat/completions").mock(side_effect=waiting) + with ledger(url) as instance: + task = asyncio.create_task(ModelRouter(settings, instance).complete_json(Purpose.HINT, [], Answer, session_id="alice:s")) + await asyncio.wait_for(called.wait(), timeout=5) + reserved = instance.project_spend + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + receipt = instance.database.one("SELECT record FROM quack_model_usage")["record"] + assert receipt["error_category"] == "cancelled" + assert receipt["estimated_usd"] == pytest.approx(reserved / MAX_ATTEMPTS) + with ledger(url) as restarted: + assert restarted.project_spend == pytest.approx(reserved) + assert restarted.session_spend("alice:rotated") == pytest.approx(reserved) + + +@respx.mock +async def test_failed_durable_reservation_never_contacts_provider(url, settings): # type: ignore[no-untyped-def] + route = respx.post("https://api.test/v1/chat/completions").mock(return_value=completion()) + with ledger(url) as instance: + instance.database.close() + with pytest.raises(ModelError, match="ledger"): + await ModelRouter(settings, instance).complete_json(Purpose.HINT, [], Answer) + assert route.call_count == 0 + + +@pytest.mark.parametrize("bad", [-1, float("nan"), float("inf"), True, "0.1", None]) +def test_invalid_receipt_preserves_debit_and_blocks_further_spending(url, bad): # type: ignore[no-untyped-def] + with ledger(url) as instance: + with pytest.raises(BudgetExceeded), instance.reserve("alice:s", .02): + instance.record(UsageRecord("test", "low", "transcribe", "test", estimated_usd=bad)) + assert instance.project_spend == pytest.approx(.06) + assert instance.database.one("SELECT COUNT(*) AS n FROM quack_model_usage")["n"] == 0 + with pytest.raises(BudgetExceeded), instance.reserve("bob:s", .001): + pytest.fail("corrupt receipt did not block spending") diff --git a/services/api/tests/test_release_safety.py b/services/api/tests/test_release_safety.py new file mode 100644 index 0000000..2a0cc07 --- /dev/null +++ b/services/api/tests/test_release_safety.py @@ -0,0 +1,376 @@ +"""Release boundaries: bad configuration, ledger integrity, retries and private diagnostics.""" + +from __future__ import annotations + +import asyncio +import io +import json +import logging +import subprocess +import sys +import time +from dataclasses import replace +from pathlib import Path +from unittest.mock import Mock, patch + +import httpx +import pytest +import respx +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pydantic import BaseModel, ValidationError + +from quack.config import Settings +from quack.llm.ledger import BudgetExceeded, Ledger, UsageRecord +from quack.llm.router import DEFAULT_ROUTES, ModelError, ModelRouter, Purpose +from quack.main import RateLimiter, build_services, create_app +from quack.observability import JsonFormatter, RequestDiagnostics, request_id +from quack.schemas import AnalyzeResponse, Mistake +from quack.services.hint_engine import _log_failure +from quack.services.verifier_pool import _child + + +@pytest.mark.parametrize("name", [ + "budget_project_usd", "budget_session_usd", "budget_request_usd", "budget_hourly_usd", + "reading_cache_size", "line_memory_size", "verify_timeout_s", "second_reading_patience_s", + "rate_limit_per_minute", "global_rate_limit_per_minute", +]) +@pytest.mark.parametrize("value", [-1, float("nan"), float("inf")]) +def test_invalid_limits_fail_at_configuration(name: str, value: float) -> None: + with pytest.raises(ValidationError): + Settings(_env_file=None, **{name: value}) + + +@pytest.mark.parametrize("name", ["budget_project_usd", "budget_session_usd", "budget_request_usd", "budget_hourly_usd"]) +def test_zero_budget_is_an_explicit_spending_stop(name: str) -> None: + assert getattr(Settings(_env_file=None, **{name: 0}), name) == 0 + + +@pytest.mark.parametrize("name", ["rate_limit_per_minute", "global_rate_limit_per_minute", "verify_timeout_s"]) +def test_zero_rate_or_timeout_is_rejected(name: str) -> None: + with pytest.raises(ValidationError): + Settings(_env_file=None, **{name: 0}) + + +@pytest.mark.parametrize("overrides", [ + {"max_tokens": 0}, {"max_tokens": -1}, {"max_tokens": True}, {"max_tokens": 1.5}, + {"timeout_s": 0}, {"timeout_s": float("nan")}, {"timeout_s": float("inf")}, + {"model": "unpriced"}, {"reasoning_effort": "typo"}, {"retry_timeouts": "false"}, +]) +def test_invalid_model_routes_fail_before_a_request(overrides: dict) -> None: + with pytest.raises(ValueError): + replace(DEFAULT_ROUTES[Purpose.HINT], **overrides) + + +def usage(cost: float, **fields) -> UsageRecord: + return UsageRecord("m", "low", "p", "r", estimated_usd=cost, **fields) + + +@pytest.mark.parametrize("bad", [-1, float("nan"), float("inf"), True, "0.1", None]) +def test_corrupt_ledger_amount_stops_spending_atomically(tmp_path: Path, bad: object) -> None: + path = tmp_path / "usage.jsonl" + ledger = Ledger(path, 100, 100, 100) + ledger.record(usage(1)) + with path.open("a") as handle: + handle.write(json.dumps({"estimated_usd": 2}) + "\n") + handle.write(json.dumps({"estimated_usd": bad}) + "\n") + for _ in range(2): + with pytest.raises(BudgetExceeded, match="ledger"): + ledger.check("s", .01) + assert ledger.project_spend == 1 # retries cannot count the valid prefix twice + + +@pytest.mark.parametrize("record", [[], None, 5, {"session_id": []}, {"timestamp": "yesterday"}]) +def test_invalid_ledger_structure_stops_spending(tmp_path: Path, record: object) -> None: + path = tmp_path / "usage.jsonl" + ledger = Ledger(path, 100, 100, 100) + path.write_text(json.dumps(record) + "\n") + with pytest.raises(BudgetExceeded, match="ledger"): + ledger.check("s", .01) + + +@pytest.mark.parametrize("bad", [-1, float("inf"), float("nan"), True]) +def test_invalid_reservations_cannot_increase_the_budget(bad: float) -> None: + ledger = Ledger(None, 1, 1, 1) + with pytest.raises(ValueError), ledger.reserve("s", bad): + pytest.fail("reservation accepted") + assert ledger._reserved_project == 0 + assert dict(ledger._reserved_session) == {} + + +def test_reservation_is_released_on_failure_and_cancellation() -> None: + ledger = Ledger(None, 1, 1, 1) + for error in (RuntimeError, asyncio.CancelledError): + with pytest.raises(error), ledger.reserve("s", .5): + raise error() + assert ledger._reserved_project == 0 + assert dict(ledger._reserved_session) == {} + ledger.check("s", 1) + + +def test_out_of_order_timestamps_do_not_pin_expired_hourly_spend() -> None: + ledger = Ledger(None, 100, 100, 100, hourly_limit=1) + ledger.record(usage(.1)) + ledger.record(usage(.95, timestamp=time.time() - 7200)) + ledger.check("s", .8) + assert ledger.project_spend == pytest.approx(1.05) + + +def test_failed_ledger_write_counts_call_and_blocks_further_spend(tmp_path: Path) -> None: + ledger = Ledger(tmp_path / "usage.jsonl", 100, 100, 100) + with patch.object(Path, "open", side_effect=OSError("disk full")): + ledger.record(usage(.3, session_id="s")) + assert ledger.project_spend == .3 + assert ledger.session_spend("s") == .3 + with pytest.raises(BudgetExceeded, match="ledger"): + ledger.check("s", .01) + + +class Answer(BaseModel): + answer: str + + +def completion(content: str = '{"answer":"ok"}') -> httpx.Response: + return httpx.Response(200, json={ + "id": "c", "object": "chat.completion", "created": 0, "model": "muse-spark-1.3", + "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}], + # Deliberately no usage: charge the conservative fallback. + }) + + +@respx.mock +@pytest.mark.parametrize("failure", ["timeout", "invalid_output"]) +async def test_retry_must_fit_remaining_budget(settings: Settings, monkeypatch: pytest.MonkeyPatch, failure: str) -> None: + monkeypatch.setattr("quack.llm.router._retry_delay", lambda *_: 0) + ledger = Ledger(None, .06, 10, 10) + router = ModelRouter(settings, ledger) + response = httpx.ReadTimeout("slow") if failure == "timeout" else completion("not JSON") + route = respx.post("https://api.test/v1/chat/completions").mock(side_effect=[response, completion()]) + with pytest.raises(ModelError) as caught: + await router.complete_json( + Purpose.EXPLAIN, [], Answer, session_id="s", + route=replace(DEFAULT_ROUTES[Purpose.EXPLAIN], retry_timeouts=True), + ) + assert caught.value.category == "budget_exceeded" + assert route.call_count == 1 + assert ledger.project_spend > 0 + assert ledger._reserved_project == 0 + + +@respx.mock +async def test_model_cancellation_releases_reservation(settings: Settings) -> None: + ledger = Ledger(None, 1, 1, 1) + router = ModelRouter(settings, ledger) + called = asyncio.Event() + + async def waiting(_: httpx.Request) -> httpx.Response: + called.set() + await asyncio.Event().wait() + return completion() + + respx.post("https://api.test/v1/chat/completions").mock(side_effect=waiting) + task = asyncio.create_task(router.complete_json(Purpose.HINT, [], Answer, session_id="s")) + await called.wait() + assert ledger._reserved_project > 0 + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert ledger._reserved_project == 0 + assert dict(ledger._reserved_session) == {} + assert ledger.project_spend > 0 # cancellation cannot erase potentially billable work + + +def test_rate_windows_expire_at_the_boundary(monkeypatch: pytest.MonkeyPatch) -> None: + now = 100. + monkeypatch.setattr("quack.main.time.monotonic", lambda: now) + limiter = RateLimiter(1, 2) + assert limiter.allow("a") + assert not limiter.allow("a") + assert limiter.allow("b") + assert not limiter.allow("c") + now += 59.999 + assert not limiter.allow("a") + now += .001 + assert limiter.allow("a") + assert limiter.allow("c") + + +@pytest.fixture +def client(settings: Settings) -> TestClient: + return TestClient(create_app(build_services(settings))) + + +@pytest.fixture +def log_output(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> io.StringIO: + output = io.StringIO() + handler = logging.StreamHandler(output) + handler.setFormatter(JsonFormatter()) + monkeypatch.setattr(logging.getLogger("quack"), "handlers", [handler]) + return output + + +@pytest.mark.parametrize("path, method, expected", [ + ("/api/health", "GET", 200), ("/api/not-real/private-id", "GET", 404), + ("/api/analyze", "POST", 422), ("/api/health", "POST", 405), +]) +def test_api_responses_have_private_correlation_headers(client: TestClient, path: str, method: str, expected: int) -> None: + response = client.request(method, path, headers={"x-request-id": "forged-id"}) + assert response.status_code == expected + assert len(response.headers["x-request-id"]) == 32 + assert response.headers["x-request-id"] != "forged-id" + assert response.headers["cache-control"] == "no-store" + assert request_id.get() is None + + +@pytest.mark.parametrize("origin", ["http://[broken", "ftp://10.0.0.1", "http://name@10.0.0.1", "null"]) +def test_malformed_or_untrusted_origin_is_refused(client: TestClient, origin: str) -> None: + response = client.post("/api/analyze", json={}, headers={"origin": origin, "x-forwarded-host": "10.0.0.1"}) + assert response.status_code == 403 + assert response.headers["x-request-id"] + + +def test_request_logs_never_include_private_url_or_body(client: TestClient, log_output: io.StringIO) -> None: + response = client.post( + "/api/analyze?secret=private-query", json={"image": "private-image"}, + headers={"authorization": "Bearer private-token"}, + ) + events = [json.loads(line) for line in log_output.getvalue().splitlines() if line.startswith("{")] + event = next(item for item in events if item["event"] == "http_request") + assert event["request_id"] == response.headers["x-request-id"] + assert event["route"] == "/api/analyze" + assert event["duration_ms"] >= 0 + assert event["status"] == 422 + assert "private-" not in json.dumps(events) + + +def test_unhandled_exception_has_safe_response_and_diagnostic(log_output: io.StringIO) -> None: + app = FastAPI() + app.add_middleware(RequestDiagnostics) + + @app.get("/api/private/{user_id}") + def broken(user_id: str) -> None: + raise RuntimeError("credential-and-handwriting") + + response = TestClient(app).get("/api/private/user-secret") + assert response.status_code == 500 + assert response.json() == {"detail": "internal server error"} + assert response.headers["x-request-id"] + output = log_output.getvalue() + events = [json.loads(line) for line in output.splitlines()] + failure = next(event for event in events if event["event"] == "request_failed") + assert failure["error_type"] == "RuntimeError" + assert failure["frames"][-1]["function"] == "broken" + assert failure["frames"][-1]["line"] > 0 + assert failure["request_id"] == response.headers["x-request-id"] + assert "credential-and-handwriting" not in output + assert "user-secret" not in output + assert request_id.get() is None + + +async def test_concurrent_requests_keep_distinct_log_contexts() -> None: + app = FastAPI() + app.add_middleware(RequestDiagnostics) + both = asyncio.Event() + ids: list[str | None] = [] + + @app.get("/api/check") + async def check() -> dict: + current = request_id.get() + ids.append(current) + if len(ids) == 2: + both.set() + await asyncio.wait_for(both.wait(), timeout=2) + assert request_id.get() == current + return {"id": current} + + async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://test") as client: + replies = await asyncio.gather(client.get("/api/check"), client.get("/api/check")) + assert len(set(ids)) == 2 + assert all(reply.json()["id"] == reply.headers["x-request-id"] for reply in replies) + assert request_id.get() is None + + +def test_json_log_format_includes_timestamp_and_structured_fields() -> None: + record = logging.makeLogRecord({"name": "quack.test", "msg": "test_event", "levelname": "INFO", "count": 3}) + result = json.loads(JsonFormatter().format(record)) + assert result["event"] == "test_event" + assert result["count"] == 3 + assert result["timestamp"].endswith("+00:00") + + +def mistake(**overrides) -> Mistake: + return Mistake(**{ + "id": "m1", "category": "arithmetic", "error_type": "arithmetic_error", + "operation_type": "arithmetic", "previous_step": "2 * 3", "current_step": "5", + "verified": True, "is_new": True, **overrides, + }) + + +@pytest.mark.parametrize("fields", [ + {"intervention": {"shouldQuack": True}}, + {"mistakes": [mistake(verified=False)]}, + {"mistakes": [mistake(), mistake()]}, + {"mistake": mistake()}, + {"mistakes": [mistake()], "mistake": mistake(verified=False)}, + {"mistakes": [mistake()], "mistake": mistake(current_step="different evidence")}, + {"mistakes": [mistake()], "resolvedMistakeIds": ["m1"]}, + {"mistakes": [mistake()], "status": "no_error", "intervention": {"shouldQuack": True}}, + {"mistakes": [mistake()], "stale": True, "intervention": {"shouldQuack": True}}, + {"mistakes": [mistake(is_new=False)], "intervention": {"shouldQuack": True}}, +]) +def test_unsafe_alert_contract_is_rejected(fields: dict) -> None: + with pytest.raises(ValidationError): + AnalyzeResponse.model_validate({"snapshotVersion": 1, "status": "confirmed_error", **fields}) + + +def test_valid_alert_and_silent_acknowledgement_pass_the_contract() -> None: + alert = AnalyzeResponse( + snapshot_version=1, status="confirmed_error", mistakes=[mistake()], mistake=mistake(), + intervention={"shouldQuack": True}, + ) + assert alert.intervention.should_quack + acknowledged = AnalyzeResponse( + snapshot_version=2, status="confirmed_error", mistakes=[mistake(is_new=False)], + ) + assert not acknowledged.intervention.should_quack + + +def test_alert_validation_remains_active_with_python_assertions_disabled() -> None: + result = subprocess.run([ + sys.executable, "-O", "-c", + "from quack.schemas import AnalyzeResponse\n" + "from pydantic import ValidationError\n" + "try:\n" + " AnalyzeResponse(snapshot_version=1, status='no_error', intervention={'shouldQuack': True})\n" + "except ValidationError:\n" + " raise SystemExit(0)\n" + "raise SystemExit('unsafe alert passed optimized Python')\n", + ], capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stderr + + +def test_cors_exposes_server_generated_request_id(client: TestClient, settings: Settings) -> None: + response = client.get("/api/health", headers={"origin": settings.cors_origin_list[0]}) + assert response.headers["access-control-allow-origin"] == settings.cors_origin_list[0] + assert "x-request-id" in response.headers["access-control-expose-headers"] + assert len(response.headers["x-request-id"]) == 32 + + +def test_background_failures_never_log_private_expressions(log_output: io.StringIO) -> None: + def broken() -> None: + raise ValueError("private-student-expression") + + connection = Mock() + with patch("quack.services.verifier_pool._limit_resources"): + _child(connection, broken) + assert connection.send.call_args_list[-1].args == (("error", "ValueError"),) + assert "private-student-expression" not in str(connection.mock_calls) + connection.close.assert_called_once() + task = Mock() + task.cancelled.return_value = False + task.exception.return_value = ValueError("private-student-expression") + _log_failure(task) + event = json.loads(log_output.getvalue().splitlines()[-1]) + assert event["event"] == "hint_preparation_failed" + assert event["error_type"] == "ValueError" + assert "private-student-expression" not in log_output.getvalue() diff --git a/services/api/tests/test_routing_regression.py b/services/api/tests/test_routing_regression.py index b6354a9..fbc3882 100644 --- a/services/api/tests/test_routing_regression.py +++ b/services/api/tests/test_routing_regression.py @@ -15,12 +15,11 @@ from quack.llm.ledger import Ledger from quack.llm.router import ModelRouter, Purpose -RESULTS = Path(__file__).resolve().parents[3] / "evals" / "results" +RESULTS = Path(__file__).parent / "fixtures" / "recorded-routing.json" def recorded(run: str, config: str | None = None) -> list[dict]: - with (RESULTS / run / "results.jsonl").open() as handle: - rows = [json.loads(line) for line in handle if line.strip()] + rows = json.loads(RESULTS.read_text())["runs"][run] return [row for row in rows if config is None or row["config"] == config] From 80083b74fc1959b1339b47eda6a3c814108637d7 Mon Sep 17 00:00:00 2001 From: Brendan-Lim <121050815+Brendan-Lim@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:00:58 +0800 Subject: [PATCH 2/2] Require every browser journey and preserve complete release evidence --- .github/workflows/verify.yml | 2 +- apps/web/playwright.config.ts | 1 + docs/RELEASE_SAFETY.md | 3 +++ scripts/check_browser_results.py | 24 ++++++++++++++++++++++++ scripts/check_release.py | 8 +++++--- scripts/public-files.txt | 1 + scripts/verify.py | 7 +++++++ 7 files changed, 42 insertions(+), 4 deletions(-) create mode 100755 scripts/check_browser_results.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 02ebe36..63c606d 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -16,7 +16,7 @@ concurrency: jobs: verify: runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 services: postgres: image: postgres:18 diff --git a/apps/web/playwright.config.ts b/apps/web/playwright.config.ts index e6ba8b9..97f2abd 100644 --- a/apps/web/playwright.config.ts +++ b/apps/web/playwright.config.ts @@ -16,6 +16,7 @@ export default defineConfig({ fullyParallel: false, forbidOnly: true, retries: 0, + outputDir: "test-results/journeys", reporter: [["list"], ["json", { outputFile: "test-results/results.json" }]], use: { ...devices["Desktop Chrome"], viewport: { width: 1280, height: 800 }, trace: "retain-on-failure", screenshot: "only-on-failure" }, projects: [ diff --git a/docs/RELEASE_SAFETY.md b/docs/RELEASE_SAFETY.md index 878b04e..440fa86 100644 --- a/docs/RELEASE_SAFETY.md +++ b/docs/RELEASE_SAFETY.md @@ -24,6 +24,9 @@ reachable publication history, staged and historical secrets, Ruff, mypy, offlin ESLint, TypeScript, frontend units, the production build, compiled browser checks and all existing browser journeys. It refuses skipped backend tests and unknown pytest settings or markers. The two paid live tests are deliberately deselected. No model credits are needed. +Release browser verification enables the existing performance and screenshot journeys as well. +JSON reports must contain passing cases and zero skipped, failed or flaky cases. Compiled and +development-server evidence uses separate output directories so the latter cannot erase the former. Historical prompt comparisons use a classification-only fixture checked against the published synthetic dataset. It contains no raw model text, usage records, images or private evaluation paths. diff --git a/scripts/check_browser_results.py b/scripts/check_browser_results.py new file mode 100755 index 0000000..4e53b40 --- /dev/null +++ b/scripts/check_browser_results.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Fail release verification if Playwright omitted cases or reported retries/failures.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + + +def main() -> int: + if len(sys.argv) != 2: + raise SystemExit("usage: check_browser_results.py REPORT.json") + report = json.loads(Path(sys.argv[1]).read_text()) + stats = report["stats"] + if report.get("errors") or stats["expected"] <= 0 or any(stats[key] for key in ("skipped", "unexpected", "flaky")): + print(f"Incomplete browser verification: {stats}") + return 1 + print(f"Browser completeness: {stats['expected']} passed, zero skipped, failed or flaky") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_release.py b/scripts/check_release.py index 0d58e22..67c3e36 100644 --- a/scripts/check_release.py +++ b/scripts/check_release.py @@ -19,9 +19,11 @@ def main() -> int: if not path.is_file(): errors.append(f"Missing published file: {name}") continue - if "/tests/" in name and path.suffix in {".ts", ".tsx", ".py"}: - if re.search(r"\b(?:test|it|describe)\s*\.\s*only\s*[(<]", path.read_text()): - errors.append(f"Focused test committed: {name}") + if ( + "/tests/" in name and path.suffix in {".ts", ".tsx", ".py"} + and re.search(r"\b(?:test|it|describe)\s*\.\s*only\s*[(<]", path.read_text()) + ): + errors.append(f"Focused test committed: {name}") package = json.loads((ROOT / "apps/web/package.json").read_text()) locked = json.loads((ROOT / "apps/web/package-lock.json").read_text())["packages"][""] for group in ("dependencies", "devDependencies"): diff --git a/scripts/public-files.txt b/scripts/public-files.txt index b636bb4..dfe2a9f 100644 --- a/scripts/public-files.txt +++ b/scripts/public-files.txt @@ -440,3 +440,4 @@ services/api/quack/observability.py services/api/requirements-tooling.lock services/api/tests/test_release_safety.py services/api/tests/fixtures/recorded-routing.json +scripts/check_browser_results.py diff --git a/scripts/verify.py b/scripts/verify.py index 2c3e754..87e5a4c 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -32,6 +32,9 @@ def main() -> int: parser.add_argument("--build", action="store_true", help="also compile the production web app") args = parser.parse_args() env = test_environment() + if args.browser: + # Existing visual/performance journeys are opt-in during development, required for release. + env.update(QUACK_PERF="1", QUACK_SCREENSHOTS="1") if not env.get("QUACK_LEDGER_TEST_DATABASE_URL"): parser.error("set QUACK_LEDGER_TEST_DATABASE_URL to a disposable local quack_ledger_test database") gitleaks = env.get("QUACK_GITLEAKS") or shutil.which("gitleaks") @@ -56,8 +59,12 @@ def main() -> int: checks.append(("Production build", "apps/web", ["npm", "run", "build"])) if args.browser: checks.append(("Compiled app safety", "apps/web", ["npm", "run", "test:production"])) + checks.append(("Compiled browser completeness", ".", [python, "scripts/check_browser_results.py", + "apps/web/test-results/production-results.json"])) if args.browser: checks.append(("Browser journeys (fake model)", "apps/web", ["npm", "run", "test:e2e"])) + checks.append(("Browser journey completeness", ".", [python, "scripts/check_browser_results.py", + "apps/web/test-results/results.json"])) for label, directory, command in checks: print(f"\n{label}", flush=True) result = subprocess.run(command, cwd=ROOT / directory, env=env, check=False)