From 1d8a1e1c663557251a0961817c25ad4a3eecf7b4 Mon Sep 17 00:00:00 2001 From: ohm Date: Wed, 5 Aug 2026 19:58:33 +0700 Subject: [PATCH 01/24] Add FastForm design spec Design for a Google Form speed-submission system: reactive daemon over a parse/compile + transport seam, stdlib-only hot path, cookie-based auth with no browser in the fire path, warm TLS connection pool, structural (not locale-dependent) response classification, and a check-then-act retry policy. Includes the calibration experiment set against self-owned test forms and a phased build order gated on cookie-only POST working. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- .../specs/2026-08-05-fastform-design.md | 401 ++++++++++++++++++ 1 file changed, 401 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-05-fastform-design.md diff --git a/docs/superpowers/specs/2026-08-05-fastform-design.md b/docs/superpowers/specs/2026-08-05-fastform-design.md new file mode 100644 index 0000000..0edb9c8 --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-fastform-design.md @@ -0,0 +1,401 @@ +# FastForm — Google Form speed-submission system + +**Date:** 2026-08-05 +**Status:** Approved design, not yet implemented + +## Problem + +A competition distributes a Google Form URL through an unknown channel at an +unknown time. Roughly 1,000 people compete; the first three valid submissions +win. Google sign-in is required. The goal is to minimize elapsed time between +the URL reaching the operator and a valid response being recorded, running from +one machine on a residential connection. + +Nothing about the target form is known in advance: not the URL, not the opening +time, not the question set. No practice replica is available. The system must +therefore discover everything at runtime and keep every assumption in config. + +## Scope boundary: knowable vs. unknowable + +The design splits along what can be learned before the event. + +**Knowable now, by experiment against forms we own.** Auth and cookie mechanics; +what success, validation-rejection, closed-form, and already-responded responses +look like; whether a closed-form POST is a no-op; whether a second POST +overwrites the first; `pageHistory` behavior across sections; `fbzx` staleness; +connection warm-up latency and jitter; rate-limit thresholds. None of this +depends on the target form. + +**Unknowable until the URL lands, therefore pure runtime discovery.** Entry IDs, +question text, question types, required flags, section count, whether sign-in is +required, whether limit-to-one is on, whether a file upload exists. Nothing here +is pre-baked. + +## Design decisions and their rationale + +### Clock sync is not the problem; open-detection is + +Google Forms has no native scheduled open. An announced opening time means a +human toggles "accepting responses" (seconds of slop), an Apps Script trigger +fires (minute granularity, 15–45s drift is routine), or the form was already +open and the announced time is just when the link drops. In none of these cases +does millisecond clock accuracy help. The system polls to detect open rather +than firing on a clock. + +### A closed form may not be parseable + +If the organizer opens by toggling, a GET before open returns the "not accepting +responses" page with no `FB_PUBLIC_LOAD_DATA_` — no entry IDs, no structure. The +payload cannot be compiled. This forces a slower path (poll GET until it renders, +then arm, then POST — roughly two extra round trips) and no part of the design +can avoid it. Acknowledged, not solved. + +### No browser in the hot path + +Authentication needs cookies, not a browser. Cookies are extracted from Chrome +once, offline; the fire path is one stdlib HTTPS POST with a Cookie header. This +removes the largest dependency and eliminates a hard failure mode: +`launch_persistent_context` fails outright if Chrome is already running on the +target profile, which it will be on the day. + +### Connection warm-up is the largest deterministic win + +Cold DNS + TCP + TLS to `docs.google.com` on residential is 150–350ms with high +variance — likely larger and jitterier than everything else combined. The system +holds 2–3 warm TLS connections with a keep-alive heartbeat and fires on an +already-handshaken socket. Multiple sockets give microsecond failover instead of +a fresh handshake when a send raises. + +### Classify by structure, not by string + +Google renders these pages in the account's UI language. Matching on +"Your response has been recorded" is fragile. Primary signals are structural: +did the response re-render a form (`FB_PUBLIC_LOAD_DATA_` present again → +rejected), did it land on the confirmation route, does a follow-up GET show the +already-responded state. Locale strings are a secondary signal, kept in a config +file populated during calibration. + +### The browser is a free parallel fallback + +The instant a URL arrives, before anything else, shell out to `open `. The +tab loads in parallel with parsing, costs one line and no dependency, and +carries no duplicate risk because the tab does nothing until touched. The +operator arbitrates between the automated path and the rendered tab. + +### Stage the fallback, do not race it + +The POST resolves in 200–400ms; a manual browser fill takes 2–5s. Firing the +browser 400ms later costs about 10% of the fallback's own latency and removes +duplicate risk entirely. Sequential with an aggressive timeout, not parallel. + +## Architecture + +Single long-running Python process (a reactive daemon) built over a two-script +seam: parse/compile and transport remain independently runnable, so if the +daemon misbehaves the operator runs two commands by hand and loses automation, +not capability. + +``` +fastform/ + config/ + answers.toml # answer bank: text patterns -> values + profile.toml # timeouts, poll intervals, strategy, gate threshold + classify.toml # response markers (emitted by calibration, not hand-written) + cookies.txt # extracted Google auth cookies (gitignored) + fastform/ + cookies.py # Chrome profile -> cookie header; freshness check + parse.py # URL or HTML bytes -> FormSpec + match.py # FormSpec + answer bank -> FilledForm (+ gaps, confidence) + compile.py # FilledForm -> raw HTTP request bytes + transport.py # warm connection pool, keep-alive, send, timing + classify.py # response bytes -> Outcome + strategy.py # the fire loop and its stop conditions + intake.py # clipboard / stdin / LAN -> one URL queue + tui.py # state display + confirm gate + daemon.py # state machine, wires the above + tools/ + calibrate.py # experiments against our own forms -> classify.toml + dryrun.py # build request, print it, do not send +``` + +### Data types + +- **`FormSpec`** — form id, action URL, page count, sign-in required, + limit-to-one, and `Question(entry_id, text, type, required, options, + validation)` list. Produced only by `parse.py`. Pure data, no network. +- **`FilledForm`** — `entry.N=value` pairs, `gaps` (required questions nothing + matched), `blockers` (file uploads, unsupported types), confidence score. +- **`RequestBlob`** — literal bytes for the socket, plus host and port. Nothing + downstream of this knows what a form is. +- **`Outcome`** — `RECORDED | REJECTED | CLOSED | ALREADY_RESPONDED | + AUTH_FAIL | UNKNOWN`. `classify.py` is a pure function from response bytes to + this enum, so it tests trivially against saved fixtures. + +### Module boundaries + +`parse` makes no policy decisions. `transport` does not know what a form is. +`classify` is pure. `strategy` is the only module that decides whether to send +again — every "will this double-submit me" question lives in one readable file. + +### Dependencies + +Hot path is stdlib only: `http.client`, `ssl`, `socket`, `json`, `re`, +`difflib`, `threading`, `http.server`, `tomllib`. No web framework, no database, +no ORM. Config is three TOML files plus a cookie file. + +The one place stdlib fights us is the clipboard: `pbpaste` forks a subprocess per +poll, so polling it at 10ms is wasteful and jittery. Primary approach is a ~30 +line `ctypes` binding to `NSPasteboard.changeCount` (an integer read, effectively +free at 5ms polling), keeping the project at zero dependencies. If that binding +proves troublesome during Phase 7, the fallback order is +`pyobjc-framework-Cocoa` as a single dependency, then `pbpaste` at 25ms as a +degraded no-dependency mode. + +## Runtime behavior + +### State machine + +`WAITING → ARMING → ARMED → FIRING → SETTLED`, with connection warm-up running +as a background service throughout. + +**Boot** happens hours early and is where failures belong: load config, load +cookies, then perform a real authenticated GET and confirm the account email +comes back. Dead cookies found at T-2h are an inconvenience; found at T+0 they +are the loss. + +**Warm pool** maintains 2–3 open TLS connections to `docs.google.com` and to +`forms.gle` (the URL may arrive shortened), heartbeating every 5–10s. + +### Intake + +Three channels feed one queue in `intake.py`, all optional and independently +toggleable, because the delivery channel is unknown: + +- **stdin** — the operator types or pastes into the waiting prompt. Covers the + "URL read out verbally" case. Always available; the only channel in the MVP. +- **clipboard** — `NSPasteboard.changeCount` polled at 5ms, so copying the URL + is the operator's only action. Any URL-shaped string on the clipboard arms the + system. +- **LAN listener** — `http.server` in a thread with a single endpoint, paired + with an iOS share-sheet Shortcut, for the case where the URL lands on a phone. + +Whichever channel fires first wins; the others are ignored for the remainder of +the run. All three normalize through the same path described below. + +### ARMING + +First action on URL receipt, before anything else: `open `, fire and forget. + +Then: normalize the URL (strip trailing punctuation from a chat paste, unwrap +redirect wrappers, resolve `forms.gle` on the warm socket, handle both +`/d/e//viewform` and the rarer `/d//` published path, drop `?usp=`), GET +over a warm socket, parse, match, compile. + +The GET has four outcomes: + +| Response | Action | +|---|---| +| Form renders | Parse, match, compile. Full speed available. | +| Closed page | No entry IDs, cannot compile. Enter `PRE_OPEN`: GET-poll until it renders, then arm, then fire. | +| Sign-in redirect | Cookies dead. Stop loud, hand to the browser tab. | +| 404 | Bad URL. Return to `WAITING`, keep listening. | + +### Matching + +Confidence ladder, highest first: + +1. Explicit regex from `answers.toml` matches question text → 1.0 +2. Exact normalized equality (casefolded, depunctuated, whitespace-collapsed) → 0.95 +3. All pattern tokens present in the question text → 0.8 +4. `difflib.SequenceMatcher` ratio above threshold → the ratio itself +5. No match → gap + +For choice-type questions the *value* must also match an entry in the option +list. An answer of "Bangkok" against an option of "Bangkok Metropolitan Region" +is a silent-rejection generator, so a value matching no option counts as a gap +even when the question matched perfectly. + +Auto-fire requires `confidence >= threshold AND no gaps AND no blockers`. +Everything else goes to the gate. + +### The gate + +Full-screen terminal render: every question, matched value, confidence, gaps in +red. Choice gaps get numbered options so one keypress fills them; free-text is +typed. Bare `Enter` fires; `b` abandons to the already-loaded browser tab. Raw +tty so single keypresses register without Enter. Renders in under 50ms, +pre-rendered as far as possible during ARMING. + +Policy on unknown required questions is config (`unknown_required` = +`prompt` | `first_valid_option` | `abort`), defaulting to `prompt`. Required free +text is never auto-guessed under any policy — `first_valid_option` applies only +to choice-type questions, and falls back to `prompt` for free text. + +Unmatched **non-required** questions are omitted from the payload entirely rather +than filled with a guess. They cannot cause a rejection, and omitting them keeps +the payload minimal. + +### FIRING + +Three modes in `strategy.py`, selected at runtime from what `parse.py` learned +rather than guessed in config: + +- **`single`** — one POST, classify, done. +- **`poll_get_then_post`** — forced when the form was closed at arming. +- **`optimistic`** — POST every `poll_ms` from a start time until terminal. + Saves a full round trip by eliminating the detect→fire gap. Available only + when the form parsed successfully, and only if calibration confirms a + closed-form POST is a no-op. + +Stop conditions: + +| Outcome | Action | +|---|---| +| `RECORDED` | Hard stop. No further sends under any circumstance. | +| `ALREADY_RESPONDED` | Stop. Treated as success. | +| `REJECTED` | Stop. Payload is wrong; repeating will not help. Surface to gate. | +| `CLOSED` | Keep polling — expected pre-open state. | +| `AUTH_FAIL` | Stop loud, hand to browser. | +| timeout / socket error | Probe, never blind-retry: GET on a second warm socket. `ALREADY_RESPONDED` means recorded; otherwise safe to resend. | + +Plus a hard `max_sends` cap and a wall-clock deadline, both config. + +The probe is definitive only when limit-to-one is on. `parse.py` already +established whether it is, so `strategy` branches on fact rather than on a +config guess. + +### Jitter control + +- `gc.disable()` on entering ARMED, re-enabled at SETTLED. +- Blob fully materialized at arm time; the send path allocates nothing. +- Fire on the main thread; intake threads sit blocked on I/O. +- Stage timestamps via `perf_counter_ns()` into a preallocated list, flushed + after SETTLED. No logging I/O in the hot path. +- Run under `caffeinate -dimsu` in a foreground terminal to avoid App Nap and + timer coalescing. +- Ethernet where available. Wi-Fi power-save adds 100ms+ to the first packet + after idle and cannot be fixed in software. + +Every failure path terminates in either a definitive `Outcome` or the +already-loaded browser tab. Nothing degrades silently. + +## Testing and calibration + +### Test bed (Phase 0, operator-created) + +Four forms in our own Google account, chosen to cover branches rather than to be +realistic: + +- **F1** — one section, sign-in required, **limit-to-1 ON**, three required + short answers. Auth and overwrite testbed. +- **F2** — identical but **limit-to-1 OFF**. Duplicate semantics; load-test + target, since extra rows are harmless. +- **F3** — three sections, mixed types (short answer, paragraph, multiple choice + with "Other", checkboxes, dropdown, linear scale, date, time), mixed required + flags, one section branch. `pageHistory` testbed. +- **F4** — regex validation on a short answer, a number range, email format, + length limits, plus one **file upload** question. Rejection-path and + blocker-detection testbed. + +### Experiments + +Each emits fixtures and populates config. + +- **E1 — cookie-only POST works?** Submit to F1 with nothing but a Cookie + header; confirm the row lands in the responses sheet under the account. This + is the go/no-go for the entire approach. On failure, diff against a DevTools + "copy as cURL" to find what is missing. +- **E2 — response fingerprints.** Capture full response bytes for success, + validation-rejected, closed, already-responded, and dead-cookies. These become + `classify.py`'s unit tests. Record status codes, redirects, and whether + `FB_PUBLIC_LOAD_DATA_` reappears. Note locale strings without depending on them. +- **E3 — overwrite semantics.** On F1, POST successfully then POST again: one + row or two, does the timestamp move? Repeat on F2. Gates optimistic mode. +- **E4 — is a closed-form POST a no-op?** Toggle a form closed, POST, reopen, + verify nothing from the closed window appears. Optimistic polling is unsafe + without this. +- **E5 — `pageHistory` on F3.** Does one POST with `0,1,2` satisfy all sections? + What happens when it is wrong or short? Does the branch change what is accepted? +- **E6 — `fbzx` staleness.** Arm from a page load, wait two hours, fire — still + accepted? Does a self-generated random `fbzx` work, or must it come from the + page? Determines whether arming hours early is a real capability. +- **E7 — latency and jitter profile.** Fifty submissions to F2, cold socket vs. + warm, recording p50/p95/max. Produces the actual timeout and `max_sends` + values instead of guesses. +- **E8 — rate limiting.** Poll GET at the intended interval for 60s. Any 429 or + captcha? Sets a safe `poll_ms`. + +### Highest-value single test + +Manually submit F3 in Chrome with DevTools open, "copy as cURL", then run +`tools/dryrun.py` against the same form and diff the two requests field by +field. This catches every missing token at once. It must be a repeatable script, +re-run after every parser change. + +### Unit tests + +All pure-function-against-fixture: `parse.py` on saved HTML, `classify.py` on +saved responses, `match.py` on synthetic `FormSpec`s built to be nasty +(near-duplicate question text, a value matching no option, an unmatched required +question). + +### Rehearsal + +End-to-end against F3 with a stopwatch, both scenarios (URL arrives early, URL +arrives at open), repeated until the gate keystrokes are muscle memory. At least +one rehearsal with something deliberately broken, so the degraded path has been +practiced rather than met for the first time on the day. + +## Build order + +| Phase | Contents | Runnable result | +|---|---|---| +| 0 | Create F1–F4, capture a manual cURL | — | +| 1 | `cookies.py` + a 20-line hardcoded POST to F1 | **E1 go/no-go gate** | +| 2 | `parse.py` + fixtures + tests | `parse ` prints FormSpec | +| 3 | `match.py`, `answers.toml`, `compile.py`, `dryrun.py` | URL → request bytes; run the cURL diff | +| 4 | `classify.py` + E2 fixtures, `transport.py` warm pool | Two-script seam complete, usable by hand | +| 5 | Calibration harness: E3–E8 | emits `classify.toml`, measured `profile.toml` | +| 6 | `strategy.py` with the stop-condition table | driven by Phase 5 findings | +| 7 | `intake.py` (stdin → clipboard → LAN), `tui.py`, `daemon.py` | the daemon | +| 8 | Rehearsals | — | + +**Phase 1 is a hard gate.** If cookie-only POST does not work the design changes +fundamentally and everything after Phase 1 is wasted. Build nothing past it +until E1 passes. + +**Minimum shippable version is Phases 1–4**: paste a URL in a shell, get a +submission in one to two seconds including typing. Phases 5–8 make it faster and +safer, not functional. + +## Out of scope + +Deliberately not built: + +- Playwright or any browser automation. The browser fallback is a tab the + operator drives. +- Millisecond clock synchronization. The opening event is not clock-driven. +- A Chrome extension. +- Headless-browser fallback. +- Retry-with-backoff in any form. Retry is check-then-act or nothing. + +Relative to the Phase 1–4 MVP, these are gold-plating and may be dropped if time +runs short: the LAN listener (stdin works), the ctypes clipboard binding (stdin +works), the multi-socket warm pool (one socket works), optimistic polling +(single-shot works), the TUI (a printed table and `input()` works). + +## Known risks + +- **Cookie-only POST may not be accepted.** Mitigated by making it Phase 1 and a + hard gate rather than an assumption. +- **The form may be closed and unparseable when the URL arrives**, forcing the + slow path. Structural, unfixable, and affects every competitor equally. +- **The answer bank may not match the real question text**, producing gaps. The + gate exists precisely for this and must be rehearsed. +- **A file upload question breaks the direct-POST path entirely.** Detected at + parse time as a blocker and routed to the browser. Detection is cheap; solving + it is not, and would lose the race anyway. +- **Google cookies may hit a "verify it's you" challenge** on the day. Mitigated + by the boot-time authenticated GET, run well before the event. +- **Organizer rules may forbid automated submission.** A disqualification risk + independent of any technical decision here; the operator has accepted it. From 5b3504fc5d9a14e03a1f7a993d7e4a83669a51b2 Mon Sep 17 00:00:00 2001 From: ohm Date: Wed, 5 Aug 2026 20:07:09 +0700 Subject: [PATCH 02/24] Fix spec inconsistencies found in review - Collapse poll_get_then_post into the PRE_OPEN sub-state; they described the same behavior, and poll_get_then_post could never execute. Add PRE_OPEN to the state machine list. - Add UNKNOWN to the stop-condition table: stop, probe once, never resend on an unclassifiable response. - Specify the limit-to-one=OFF timeout branch, previously an open case: resend directly without probing, since no already-responded state exists to observe. - Route blockers to browser handoff rather than the gate, resolving a contradiction between the runtime and risks sections. - Stop intake threads on entering ARMED; the 5ms clipboard poll is a spin loop, not a blocked read, and contended with the fire window. - Replace blanket gc.disable() with gc.freeze() at ARMED plus disable only around each send, since PRE_OPEN can run for minutes. - Add E9: verify warm TLS connections survive hours of idle. The warm-pool premise rested on this and nothing tested it. - Correct E7 to set the send timeout from p95; max_sends derives from E3. - Make Phase 4's sender explicitly single-shot so the first usable version is safe by construction, and drop the "ship tonight" framing, which assumed a deadline that does not exist. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- .../specs/2026-08-05-fastform-design.md | 99 +++++++++++++++---- 1 file changed, 78 insertions(+), 21 deletions(-) diff --git a/docs/superpowers/specs/2026-08-05-fastform-design.md b/docs/superpowers/specs/2026-08-05-fastform-design.md index 0edb9c8..86d9581 100644 --- a/docs/superpowers/specs/2026-08-05-fastform-design.md +++ b/docs/superpowers/specs/2026-08-05-fastform-design.md @@ -158,6 +158,9 @@ degraded no-dependency mode. `WAITING → ARMING → ARMED → FIRING → SETTLED`, with connection warm-up running as a background service throughout. +`ARMING` has one sub-state, `PRE_OPEN`, entered when the URL resolves to a form +that is not yet accepting responses. See ARMING below. + **Boot** happens hours early and is where failures belong: load config, load cookies, then perform a real authenticated GET and confirm the account email comes back. Dead cookies found at T-2h are an inconvenience; found at T+0 they @@ -196,10 +199,17 @@ The GET has four outcomes: | Response | Action | |---|---| | Form renders | Parse, match, compile. Full speed available. | -| Closed page | No entry IDs, cannot compile. Enter `PRE_OPEN`: GET-poll until it renders, then arm, then fire. | +| Closed page | No entry IDs, cannot compile. Enter `PRE_OPEN`. | | Sign-in redirect | Cookies dead. Stop loud, hand to the browser tab. | | 404 | Bad URL. Return to `WAITING`, keep listening. | +**`PRE_OPEN`** GET-polls at `poll_ms` until the form renders, then completes +arming (parse, match, compile) and proceeds to `FIRING`. Because the form is +known open on exit from `PRE_OPEN`, firing always uses `single` — `optimistic` +has nothing left to save. `PRE_OPEN` is the slow path, costing roughly two extra +round trips versus arriving at an already-open form, and that cost is +structural. + ### Matching Confidence ladder, highest first: @@ -216,7 +226,10 @@ is a silent-rejection generator, so a value matching no option counts as a gap even when the question matched perfectly. Auto-fire requires `confidence >= threshold AND no gaps AND no blockers`. -Everything else goes to the gate. + +Gaps go to the gate. **Blockers bypass the gate entirely** and go straight to +browser handoff — a file upload cannot be resolved with a keypress at a +terminal, and presenting it at the gate would waste the seconds that matter. ### The gate @@ -237,15 +250,15 @@ the payload minimal. ### FIRING -Three modes in `strategy.py`, selected at runtime from what `parse.py` learned +Two modes in `strategy.py`, selected at runtime from what `parse.py` learned rather than guessed in config: -- **`single`** — one POST, classify, done. -- **`poll_get_then_post`** — forced when the form was closed at arming. +- **`single`** — one POST, classify, done. Used whenever the form was already + open at arming time, and always on exit from `PRE_OPEN`. - **`optimistic`** — POST every `poll_ms` from a start time until terminal. - Saves a full round trip by eliminating the detect→fire gap. Available only - when the form parsed successfully, and only if calibration confirms a - closed-form POST is a no-op. + Saves a full round trip by eliminating the detect→fire gap. Applies only to + the case where the form parsed but is scheduled to open later, and only if + calibration (E4) confirms a closed-form POST is a no-op. Stop conditions: @@ -256,19 +269,37 @@ Stop conditions: | `REJECTED` | Stop. Payload is wrong; repeating will not help. Surface to gate. | | `CLOSED` | Keep polling — expected pre-open state. | | `AUTH_FAIL` | Stop loud, hand to browser. | -| timeout / socket error | Probe, never blind-retry: GET on a second warm socket. `ALREADY_RESPONDED` means recorded; otherwise safe to resend. | +| `UNKNOWN` | Stop sending. Probe once; if inconclusive, hand to browser with a loud warning that submission status is undetermined. Never resend on an unclassifiable response. | +| timeout / socket error | Probe, never blind-retry. Behavior depends on limit-to-one, below. | Plus a hard `max_sends` cap and a wall-clock deadline, both config. -The probe is definitive only when limit-to-one is on. `parse.py` already -established whether it is, so `strategy` branches on fact rather than on a -config guess. +**Timeout behavior branches on limit-to-one**, which `parse.py` already +established, so `strategy` branches on fact rather than a config guess: + +- **Limit-to-one ON** — GET-probe on a second warm socket. `ALREADY_RESPONDED` + means the send landed; stop. Anything else means it did not; resend. +- **Limit-to-one OFF** — the probe cannot distinguish recorded from lost, because + there is no already-responded state to observe. Resend directly, up to + `max_sends`. A duplicate row is acceptable here: the organizer takes the + earliest qualifying response, so an extra row costs nothing while a lost + submission costs everything. Skipping the probe also saves a round trip. + +`UNKNOWN` is deliberately more conservative than a timeout. A timeout means the +send may not have landed; an unclassifiable response means it may well have, and +we cannot tell what state we are in. ### Jitter control -- `gc.disable()` on entering ARMED, re-enabled at SETTLED. +- `gc.freeze()` on entering ARMED, moving everything allocated so far into the + permanent generation. `gc.disable()` only across the actual send window, not + from ARMED to SETTLED — in `PRE_OPEN` or a long `optimistic` run that window is + minutes long and allocates a response body per poll, so the collector stays on + between polls and is disabled only around each send. - Blob fully materialized at arm time; the send path allocates nothing. -- Fire on the main thread; intake threads sit blocked on I/O. +- Fire on the main thread. **Intake threads stop on entering ARMED** — they have + done their job, and the 5ms clipboard poll is a spin loop, not a blocked read, + so leaving it running would contend during exactly the window being protected. - Stage timestamps via `perf_counter_ns()` into a preallocated list, flushed after SETTLED. No logging I/O in the hot path. - Run under `caffeinate -dimsu` in a foreground terminal to avoid App Nap and @@ -320,10 +351,19 @@ Each emits fixtures and populates config. accepted? Does a self-generated random `fbzx` work, or must it come from the page? Determines whether arming hours early is a real capability. - **E7 — latency and jitter profile.** Fifty submissions to F2, cold socket vs. - warm, recording p50/p95/max. Produces the actual timeout and `max_sends` - values instead of guesses. + warm, recording p50/p95/max. Sets the send timeout (from p95, not p50 — the + timeout exists to catch the tail) and quantifies what warm-up actually buys. + `max_sends` is a safety cap, not a latency figure, and is set from E3's + overwrite findings instead. - **E8 — rate limiting.** Poll GET at the intended interval for 60s. Any 429 or captcha? Sets a safe `poll_ms`. +- **E9 — warm connection survival.** Hold a TLS connection to `docs.google.com` + with the intended heartbeat for three hours, then fire on it and confirm the + submission lands. Google may reap idle connections, NAT mappings expire, and + consumer routers drop state. The entire warm-pool premise rests on this and + nothing else tests it. If connections do not survive, the fix is a shorter + heartbeat or periodic reconnection — determine which, and set the heartbeat + interval from the measured survival time rather than the assumed 5–10s. ### Highest-value single test @@ -354,8 +394,8 @@ practiced rather than met for the first time on the day. | 1 | `cookies.py` + a 20-line hardcoded POST to F1 | **E1 go/no-go gate** | | 2 | `parse.py` + fixtures + tests | `parse ` prints FormSpec | | 3 | `match.py`, `answers.toml`, `compile.py`, `dryrun.py` | URL → request bytes; run the cURL diff | -| 4 | `classify.py` + E2 fixtures, `transport.py` warm pool | Two-script seam complete, usable by hand | -| 5 | Calibration harness: E3–E8 | emits `classify.toml`, measured `profile.toml` | +| 4 | `classify.py` + E2 fixtures, `transport.py` warm pool, **single-shot sender** | Two-script seam complete, usable by hand | +| 5 | Calibration harness: E3–E9 | emits `classify.toml`, measured `profile.toml` | | 6 | `strategy.py` with the stop-condition table | driven by Phase 5 findings | | 7 | `intake.py` (stdin → clipboard → LAN), `tui.py`, `daemon.py` | the daemon | | 8 | Rehearsals | — | @@ -364,9 +404,26 @@ practiced rather than met for the first time on the day. fundamentally and everything after Phase 1 is wasted. Build nothing past it until E1 passes. -**Minimum shippable version is Phases 1–4**: paste a URL in a shell, get a -submission in one to two seconds including typing. Phases 5–8 make it faster and -safer, not functional. +**Phase 4's sender fires exactly once.** One POST, classify, print the outcome, +stop. It contains no loop and no resend path, so it cannot double-submit — the +safety is structural, not logical, which matters because at Phase 4 no +calibration has run and there is nothing on which to base correct retry +behavior. `strategy.py` in Phase 6 replaces it with measured retry discipline. +Until then, a timed-out send is treated as a loss and handed to the browser. + +**Phases 1–4 are the first usable version.** Paste a URL in a shell, get a +submission in roughly one to two seconds including typing. It is competitive on +its own, and it is what to fall back to if later phases are incomplete when the +event arrives. + +Its sender fires exactly once (see Phase 4 below), so it is safe by construction +rather than by logic. What Phases 5–8 add is speed and the ability to *recover*: +without `strategy.py`, a timed-out send is simply a loss, because nothing is +allowed to investigate or resend. That is an acceptable trade for a version with +no measurements behind it, but it is a real limitation, not a rough edge. + +The event date is unknown, so there is no deadline to plan against. Build in +phase order and stop wherever you are; every phase boundary is a usable state. ## Out of scope From 4719292bc80ee532b7fcae97b1e6f0ff14b45420 Mon Sep 17 00:00:00 2001 From: ohm Date: Wed, 5 Aug 2026 21:00:08 +0700 Subject: [PATCH 03/24] Add hold state; fix optimistic mode reachability Second review pass found that the "URL arrives early" scenario had no representation in the design at all: ARMING flowed straight into FIRING, so a leaked-early URL fired immediately against a form the organizer had not yet opened for competition. - Add ARMED_HOLDING sub-state and open_at config. Parsing, matching, and gap reporting happen on URL receipt; firing waits. The value is discovering an unmatched required question hours early rather than at the open bell. - Re-scope optimistic mode, which had become unreachable: the ARMING table can produce no state where the form both parses and is not yet open. It is now scoped to the one reachable case (held FormSpec + form closed for reset before open), which the hold state creates. - Hold-time re-validation, since an organizer may edit the form between leak and open and stale the entry IDs. - Intake now stops on FIRING rather than on arming, so a replacement URL can supersede a held one. Updated the jitter section to match. - Resolve profile.toml listing "strategy" as config while FIRING says modes are runtime-selected. - Note classify.py handles probe-GET bodies, since ALREADY_RESPONDED is not observable in a POST response. - Remove duplicated Phase 4 paragraph and remaining "MVP" framing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- .../specs/2026-08-05-fastform-design.md | 110 ++++++++++++++---- 1 file changed, 86 insertions(+), 24 deletions(-) diff --git a/docs/superpowers/specs/2026-08-05-fastform-design.md b/docs/superpowers/specs/2026-08-05-fastform-design.md index 86d9581..34f810c 100644 --- a/docs/superpowers/specs/2026-08-05-fastform-design.md +++ b/docs/superpowers/specs/2026-08-05-fastform-design.md @@ -99,7 +99,7 @@ not capability. fastform/ config/ answers.toml # answer bank: text patterns -> values - profile.toml # timeouts, poll intervals, strategy, gate threshold + profile.toml # timeouts, poll intervals, gate threshold, hold policy classify.toml # response markers (emitted by calibration, not hand-written) cookies.txt # extracted Google auth cookies (gitignored) fastform/ @@ -129,7 +129,9 @@ fastform/ downstream of this knows what a form is. - **`Outcome`** — `RECORDED | REJECTED | CLOSED | ALREADY_RESPONDED | AUTH_FAIL | UNKNOWN`. `classify.py` is a pure function from response bytes to - this enum, so it tests trivially against saved fixtures. + this enum, so it tests trivially against saved fixtures. It classifies both + POST responses and probe-GET bodies — `ALREADY_RESPONDED` in particular is + normally only observable on a GET, not in the POST response itself. ### Module boundaries @@ -158,8 +160,12 @@ degraded no-dependency mode. `WAITING → ARMING → ARMED → FIRING → SETTLED`, with connection warm-up running as a background service throughout. -`ARMING` has one sub-state, `PRE_OPEN`, entered when the URL resolves to a form -that is not yet accepting responses. See ARMING below. +Two sub-states: + +- **`PRE_OPEN`** (under `ARMING`) — the URL resolved to a form that is not yet + accepting responses, so there is no structure to parse. See ARMING below. +- **`ARMED_HOLDING`** (under `ARMED`) — the payload is built but `open_at` has + not yet arrived. See Holding below. **Boot** happens hours early and is where failures belong: load config, load cookies, then perform a real authenticated GET and confirm the account email @@ -175,15 +181,21 @@ Three channels feed one queue in `intake.py`, all optional and independently toggleable, because the delivery channel is unknown: - **stdin** — the operator types or pastes into the waiting prompt. Covers the - "URL read out verbally" case. Always available; the only channel in the MVP. + "URL read out verbally" case. Always available; the only channel present in + the Phase 1–4 version. - **clipboard** — `NSPasteboard.changeCount` polled at 5ms, so copying the URL is the operator's only action. Any URL-shaped string on the clipboard arms the system. - **LAN listener** — `http.server` in a thread with a single endpoint, paired with an iOS share-sheet Shortcut, for the case where the URL lands on a phone. -Whichever channel fires first wins; the others are ignored for the remainder of -the run. All three normalize through the same path described below. +Whichever channel delivers first triggers arming. Intake does **not** shut down +at that point: during `ARMED_HOLDING` a later URL that differs from the held one +supersedes it and re-arms, because an organizer replacing the form with a new +link is a real possibility and the held payload would be worthless. A repeat of +the same URL is ignored. Intake stops only on entering `FIRING`. + +All three channels normalize through the same path described below. ### ARMING @@ -210,6 +222,40 @@ has nothing left to save. `PRE_OPEN` is the slow path, costing roughly two extra round trips versus arriving at an already-open form, and that cost is structural. +On successful arming the system enters `ARMED_HOLDING` if `open_at` is set and +still in the future, and `FIRING` otherwise. + +### Holding + +If `open_at` is set in `profile.toml` and has not yet passed, arming completes +into `ARMED_HOLDING` rather than firing. This is the "URL leaked early" case, +and holding is the default because firing early is a one-way door: a submission +recorded two hours before the announced open may be discarded by the organizer, +and under limit-to-one there is no second attempt. If `open_at` is unset, +arming proceeds directly to `FIRING`. + +The point of holding is not the wait — it is that **parsing, matching, and gap +reporting all happen immediately on URL receipt**, hours before they matter. An +unmatched required question discovered at 1pm is an answer-bank edit; the same +question discovered at 3:30:00 is the loss. The gate renders as soon as arming +completes and stays on screen for the duration of the hold, showing the filled +payload, any gaps, and a countdown. + +During the hold: + +- **Gaps stay editable.** Fill them at any point; the payload recompiles. +- **The form is re-validated** on an interval. An organizer may edit the form, + close it to reset, or replace it between leak and open. If the question set + changes, entry IDs may go stale, so a re-GET that detects a structural change + re-runs matching and flags it loudly rather than firing a payload built + against a form that no longer exists. +- **A replacement URL supersedes the held one.** Intake stays live throughout + the hold; a different URL re-arms from scratch. +- **A keypress fires early**, overriding the hold, for the case where the + operator judges that the announced time is not binding. + +At `open_at`, firing proceeds automatically using whichever mode applies. + ### Matching Confidence ladder, highest first: @@ -253,12 +299,18 @@ the payload minimal. Two modes in `strategy.py`, selected at runtime from what `parse.py` learned rather than guessed in config: -- **`single`** — one POST, classify, done. Used whenever the form was already - open at arming time, and always on exit from `PRE_OPEN`. -- **`optimistic`** — POST every `poll_ms` from a start time until terminal. - Saves a full round trip by eliminating the detect→fire gap. Applies only to - the case where the form parsed but is scheduled to open later, and only if - calibration (E4) confirms a closed-form POST is a no-op. +- **`single`** — one POST, classify, done. Used whenever the form is known open + at the moment of firing: no hold was configured, or the hold expired with the + form still accepting. Always used on exit from `PRE_OPEN`, since the form is + known open there by definition. +- **`optimistic`** — POST every `poll_ms` until terminal. Reachable in exactly + one situation: we hold a parsed `FormSpec` from `ARMED_HOLDING`, and hold-time + re-validation shows the form has since closed — the organizer shut it to reset + before the announced open. Because entry IDs are already in hand, POST-polling + through the reopen saves the full round trip that `PRE_OPEN` cannot avoid. + Requires calibration (E4) confirming a closed-form POST is a no-op; without + that finding the mode is disabled and this case degrades to `PRE_OPEN` + behavior. Stop conditions: @@ -297,9 +349,11 @@ we cannot tell what state we are in. minutes long and allocates a response body per poll, so the collector stays on between polls and is disabled only around each send. - Blob fully materialized at arm time; the send path allocates nothing. -- Fire on the main thread. **Intake threads stop on entering ARMED** — they have - done their job, and the 5ms clipboard poll is a spin loop, not a blocked read, - so leaving it running would contend during exactly the window being protected. +- Fire on the main thread. **Intake threads stop on entering `FIRING`**, not on + arming — a replacement URL during `ARMED_HOLDING` must still be able to land. + Stopping them matters because the 5ms clipboard poll is a spin loop, not a + blocked read, and would otherwise contend during exactly the window being + protected. - Stage timestamps via `perf_counter_ns()` into a preallocated list, flushed after SETTLED. No logging I/O in the hot path. - Run under `caffeinate -dimsu` in a foreground terminal to avoid App Nap and @@ -397,7 +451,7 @@ practiced rather than met for the first time on the day. | 4 | `classify.py` + E2 fixtures, `transport.py` warm pool, **single-shot sender** | Two-script seam complete, usable by hand | | 5 | Calibration harness: E3–E9 | emits `classify.toml`, measured `profile.toml` | | 6 | `strategy.py` with the stop-condition table | driven by Phase 5 findings | -| 7 | `intake.py` (stdin → clipboard → LAN), `tui.py`, `daemon.py` | the daemon | +| 7 | `intake.py` (stdin → clipboard → LAN), `tui.py`, `daemon.py` incl. `ARMED_HOLDING` | the daemon | | 8 | Rehearsals | — | **Phase 1 is a hard gate.** If cookie-only POST does not work the design changes @@ -411,13 +465,16 @@ calibration has run and there is nothing on which to base correct retry behavior. `strategy.py` in Phase 6 replaces it with measured retry discipline. Until then, a timed-out send is treated as a loss and handed to the browser. +**Holding is manual before Phase 7.** `ARMED_HOLDING` lives in `daemon.py`, so +in the Phase 1–4 version the operator holds by running the parse step early and +the send step at the chosen moment. The two-script seam makes this natural, and +it preserves the property that matters most — gaps surface at parse time, hours +before firing. + **Phases 1–4 are the first usable version.** Paste a URL in a shell, get a submission in roughly one to two seconds including typing. It is competitive on its own, and it is what to fall back to if later phases are incomplete when the -event arrives. - -Its sender fires exactly once (see Phase 4 below), so it is safe by construction -rather than by logic. What Phases 5–8 add is speed and the ability to *recover*: +event arrives. What Phases 5–8 add is speed and the ability to *recover*: without `strategy.py`, a timed-out send is simply a loss, because nothing is allowed to investigate or resend. That is an acceptable trade for a version with no measurements behind it, but it is a real limitation, not a rough edge. @@ -436,9 +493,9 @@ Deliberately not built: - Headless-browser fallback. - Retry-with-backoff in any form. Retry is check-then-act or nothing. -Relative to the Phase 1–4 MVP, these are gold-plating and may be dropped if time -runs short: the LAN listener (stdin works), the ctypes clipboard binding (stdin -works), the multi-socket warm pool (one socket works), optimistic polling +Relative to the Phase 1–4 version, these are gold-plating and may be dropped if +time runs short: the LAN listener (stdin works), the ctypes clipboard binding +(stdin works), the multi-socket warm pool (one socket works), optimistic polling (single-shot works), the TUI (a printed table and `input()` works). ## Known risks @@ -454,5 +511,10 @@ works), the multi-socket warm pool (one socket works), optimistic polling it is not, and would lose the race anyway. - **Google cookies may hit a "verify it's you" challenge** on the day. Mitigated by the boot-time authenticated GET, run well before the event. +- **The form may be edited between an early leak and the announced open**, + staling the entry IDs a held payload was built against. Mitigated by hold-time + re-validation, which re-runs matching on a detected structural change. A + wholly different replacement URL is handled by intake staying live through the + hold, but only if that new URL actually reaches the operator. - **Organizer rules may forbid automated submission.** A disqualification risk independent of any technical decision here; the operator has accepted it. From 84d17ef3b74dfaa890f2e1ddbf1045a04aec0c5a Mon Sep 17 00:00:00 2001 From: ohm Date: Wed, 5 Aug 2026 21:20:25 +0700 Subject: [PATCH 04/24] Add implementation plan for spec phases 0-4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eleven TDD tasks producing a hand-run tool: paste a form URL, get a submission. Scoped to phases 0-4 deliberately — phase 6 consumes values that phase 5 calibration measures, so planning it now would mean inventing numbers. Two deviations from the spec, both to avoid speculative work: cookies are captured by hand from DevTools rather than decrypted from Chrome's SQLite store (decryption needs AES, hence a dependency), and RequestBlob holds an encoded body plus headers rather than raw socket bytes (E7 decides whether raw serialization earns its keep). All plan code was extracted and executed before committing: 40 tests pass, and the parse/match/compile pipeline was exercised end to end on synthetic form data. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- .../plans/2026-08-05-fastform-phases-0-4.md | 2331 +++++++++++++++++ 1 file changed, 2331 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md diff --git a/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md b/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md new file mode 100644 index 0000000..383b993 --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md @@ -0,0 +1,2331 @@ +# FastForm Phases 0–4 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a hand-run tool that takes a Google Form URL, parses its structure, fills it from a prepared answer bank, submits it in a single authenticated POST, and reports what happened. + +**Architecture:** Pure-function pipeline with a thin CLI over it — `parse` (HTML → `FormSpec`), `match` (`FormSpec` + answer bank → `FilledForm`), `compile` (`FilledForm` → `RequestBlob`), `transport` (warm connection, send), `classify` (response bytes → `Outcome`). Every stage is independently testable with no network. The CLI wires them together and is the only place that touches both the network and the domain types. + +**Tech Stack:** Python 3.14 (3.11+ required for `tomllib`), stdlib only. Tests via `unittest`. No pytest, no requests, no Playwright. + +**Source spec:** `docs/superpowers/specs/2026-08-05-fastform-design.md` + +## Global Constraints + +- **Stdlib only.** No third-party packages in `fastform/`. If a task seems to need one, stop and raise it. +- **Python 3.11+** (`tomllib`). Development target is 3.14.6. +- **Test runner:** `python3 -m unittest discover -s tests -v` from the repo root. +- **No network in unit tests.** Network verification is done by explicit manual commands with stated expected output. +- **`config/cookies.txt` is never committed.** It goes in `.gitignore` in Task 1. +- **Repo root:** `/Users/supa/projects/active/fastform`. All paths below are relative to it. +- **Scope:** This plan covers spec Phases 0–4 only. Phase 5 (calibration) and beyond require measurements that do not exist yet and get their own plan. + +## Scope Note + +Two deliberate deviations from the spec, both to avoid speculative work: + +1. **Cookies are captured manually** from Chrome DevTools into `config/cookies.txt`, not decrypted from Chrome's SQLite store. Decryption needs AES, which needs a dependency or an `openssl` subprocess, and Google session cookies last months. Automating this is deferred until it proves annoying. +2. **`RequestBlob` holds an encoded body string plus a headers dict**, not raw socket bytes. The costly part (URL-encoding the payload) still happens at arm time. Header serialization by `http.client` is microseconds. Spec experiment E7 decides whether raw-byte serialization is worth building. + +--- + +### Task 1: Scaffold the repo and create test form F1 + +**Files:** +- Create: `.gitignore` +- Create: `fastform/__init__.py` +- Create: `tests/__init__.py` +- Create: `config/answers.toml` +- Create: `config/cookies.txt` (untracked) +- Create: `fixtures/README.md` + +**Interfaces:** +- Consumes: nothing +- Produces: the package layout every later task imports from, and the F1 test form that Task 3's gate test targets. + +- [ ] **Step 1: Create the directory layout** + +```bash +cd /Users/supa/projects/active/fastform +mkdir -p fastform tests config fixtures +touch fastform/__init__.py tests/__init__.py +``` + +- [ ] **Step 2: Write `.gitignore`** + +Create `.gitignore`: + +``` +config/cookies.txt +__pycache__/ +*.pyc +.DS_Store +``` + +- [ ] **Step 3: Create test form F1 in Google Forms** + +This is manual work in a browser. Go to https://forms.google.com and create a form with these exact properties: + +- Title: `FastForm F1` +- Settings → Responses → **Collect email addresses: off** +- Settings → Responses → **Limit to 1 response: ON** (this forces sign-in) +- One section only. +- Three questions, all **Short answer**, all **Required**: + 1. `What is your full name?` + 2. `What is your email address?` + 3. `What is your phone number?` + +Click **Send** → link icon → copy the URL. Save it somewhere you can paste later; it looks like `https://docs.google.com/forms/d/e//viewform`. + +- [ ] **Step 4: Submit F1 once by hand and capture the request** + +Open the F1 URL in Chrome, signed into the Google account you will compete with. + +1. Open DevTools (Cmd+Opt+I) → **Network** tab. +2. Check **Preserve log**. +3. Fill in the three answers and click Submit. +4. In the Network list, find the request named `formResponse` (method POST). +5. Right-click it → **Copy** → **Copy as cURL**. +6. Paste into `fixtures/f1-manual-submit.curl.txt`. + +This file is the ground truth Task 8 diffs against. It contains your cookies, so add it to `.gitignore`: + +```bash +echo "fixtures/*.curl.txt" >> .gitignore +``` + +- [ ] **Step 5: Capture your cookie header** + +In the same DevTools Network panel, click the `formResponse` request → **Headers** → **Request Headers** → find `Cookie:`. Copy the entire value (it is long). + +Write it to `config/cookies.txt` as a single line. The leading `Cookie:` is optional — Task 2 strips it either way. + +- [ ] **Step 6: Save the form page HTML as a fixture** + +```bash +curl -s -H "Cookie: $(cat config/cookies.txt)" \ + '' > fixtures/f1-viewform.html +wc -c fixtures/f1-viewform.html +``` + +Expected: a byte count in the tens of thousands (typically 50,000–200,000). If it is under 5,000 you got a sign-in redirect page instead — recheck the cookie. + +Verify the form data is present: + +```bash +grep -c FB_PUBLIC_LOAD_DATA_ fixtures/f1-viewform.html +``` + +Expected: `1` + +- [ ] **Step 7: Write `fixtures/README.md`** + +```markdown +# Fixtures + +Captured artifacts used as test data. Nothing here is generated by code. + +- `f1-viewform.html` — F1 form page, signed in. Input for parse tests. +- `f1-manual-submit.curl.txt` — DevTools "copy as cURL" of a real manual + submission to F1. Ground truth for the Task 8 request diff. Untracked: + contains cookies. + +Response fixtures (`resp-*.bin`) are captured in Task 9. +``` + +- [ ] **Step 8: Write a starter `config/answers.toml`** + +Use values matching what you typed into F1 in Step 4, so later tasks can assert against them. + +```toml +# Answer bank. Each entry matches a question by text and supplies a value. +# match — list of phrases; scored against question text (see fastform/match.py) +# regex — optional; a hit here is an exact match, outranking everything +# value — what gets submitted + +[[answer]] +match = ["full name", "your name", "name"] +value = "Test Person" + +[[answer]] +match = ["email address", "email"] +regex = "e-?mail" +value = "test@example.com" + +[[answer]] +match = ["phone number", "phone", "mobile"] +value = "0800000000" +``` + +- [ ] **Step 9: Commit** + +```bash +git add .gitignore fastform/ tests/ config/answers.toml fixtures/README.md +git commit -m "Scaffold package layout and answer bank" +``` + +Confirm `config/cookies.txt` and `fixtures/*.curl.txt` are NOT in the commit: + +```bash +git show --stat --name-only HEAD +``` + +Expected: no `cookies.txt`, no `.curl.txt`. + +--- + +### Task 2: Cookie loading and freshness check + +**Files:** +- Create: `fastform/cookies.py` +- Create: `tests/test_cookies.py` + +**Interfaces:** +- Consumes: `config/cookies.txt` from Task 1. +- Produces: + - `load(path: Path | None = None) -> str` — returns a bare cookie header value. + - `verify(cookie_header: str) -> tuple[bool, str | None]` — `(signed_in, email_or_none)`. Network call. + - `USER_AGENT: str` — the UA string every later task sends. + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_cookies.py`: + +```python +import tempfile +import unittest +from pathlib import Path + +from fastform import cookies + + +class TestLoad(unittest.TestCase): + def _write(self, text): + tmp = Path(tempfile.mkdtemp()) / "cookies.txt" + tmp.write_text(text, encoding="utf-8") + return tmp + + def test_returns_bare_value(self): + path = self._write("SID=abc; HSID=def\n") + self.assertEqual(cookies.load(path), "SID=abc; HSID=def") + + def test_strips_cookie_prefix(self): + path = self._write("Cookie: SID=abc; HSID=def") + self.assertEqual(cookies.load(path), "SID=abc; HSID=def") + + def test_strips_lowercase_prefix(self): + path = self._write("cookie: SID=abc") + self.assertEqual(cookies.load(path), "SID=abc") + + def test_empty_file_raises(self): + path = self._write(" \n") + with self.assertRaises(ValueError): + cookies.load(path) + + def test_missing_file_raises(self): + path = Path(tempfile.mkdtemp()) / "nope.txt" + with self.assertRaises(FileNotFoundError): + cookies.load(path) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +cd /Users/supa/projects/active/fastform +python3 -m unittest tests.test_cookies -v +``` + +Expected: `ModuleNotFoundError: No module named 'fastform.cookies'` + +- [ ] **Step 3: Write the implementation** + +Create `fastform/cookies.py`: + +```python +"""Load a Cookie header captured from Chrome DevTools and check it still authenticates. + +Cookies are captured by hand rather than decrypted out of Chrome's SQLite +store: decryption needs AES (a dependency), and Google session cookies last +months. See the plan's Scope Note. +""" + +import http.client +import re +from pathlib import Path + +USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36" +) + +DEFAULT_PATH = Path(__file__).resolve().parent.parent / "config" / "cookies.txt" + +_EMAIL_RE = re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+") + + +def load(path: Path | None = None) -> str: + """Return the bare Cookie header value, with any 'Cookie:' prefix removed.""" + path = path or DEFAULT_PATH + raw = path.read_text(encoding="utf-8").strip() + if raw.lower().startswith("cookie:"): + raw = raw.split(":", 1)[1].strip() + if not raw: + raise ValueError(f"{path} is empty — paste a Cookie header into it") + return raw + + +def verify(cookie_header: str) -> tuple[bool, str | None]: + """GET a signed-in-only page. Returns (signed_in, email_if_found). + + Signed-out sessions get redirected to accounts.google.com, which is a + structural signal and does not depend on the account's UI language. + """ + conn = http.client.HTTPSConnection("docs.google.com", timeout=15) + try: + conn.request( + "GET", + "/forms/u/0/", + headers={"Cookie": cookie_header, "User-Agent": USER_AGENT}, + ) + resp = conn.getresponse() + location = resp.getheader("Location") or "" + body = resp.read() + finally: + conn.close() + + if resp.status in (301, 302, 303, 307, 308) and "accounts.google.com" in location: + return False, None + if resp.status != 200: + return False, None + + match = _EMAIL_RE.search(body.decode("utf-8", "replace")) + return True, match.group(0) if match else None +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +python3 -m unittest tests.test_cookies -v +``` + +Expected: `Ran 5 tests` … `OK` + +- [ ] **Step 5: Manually verify the cookie actually works** + +```bash +python3 -c " +from fastform import cookies +c = cookies.load() +print(cookies.verify(c)) +" +``` + +Expected: `(True, 'your.email@gmail.com')` — or `(True, None)` if no email appears in the page, which is still a pass. + +If you get `(False, None)`: the cookie is stale or was copied incompletely. Redo Task 1 Step 5. **Do not continue until this returns `True`.** + +- [ ] **Step 6: Commit** + +```bash +git add fastform/cookies.py tests/test_cookies.py +git commit -m "Add cookie loading and freshness check" +``` + +--- + +### Task 3: E1 gate — prove a cookie-only POST records a response + +**Files:** +- Create: `tools/e1_gate.py` + +**Interfaces:** +- Consumes: `fastform.cookies.load`, `fastform.cookies.USER_AGENT` from Task 2. +- Produces: a yes/no answer to whether the entire design is viable. No code downstream depends on this file. + +This task is a spike, not a component. It is deliberately hardcoded and throwaway. **If it fails, stop the project and re-plan** — every later task assumes it passed. + +- [ ] **Step 1: Read the entry IDs out of your captured cURL** + +```bash +grep -o 'entry\.[0-9]*' fixtures/f1-manual-submit.curl.txt | sort -u +``` + +Expected: three lines, e.g. `entry.1234567890`, `entry.987654321`, `entry.55555555`. + +Also get the form action path: + +```bash +grep -o '/forms/d/e/[^/]*/formResponse' fixtures/f1-manual-submit.curl.txt | head -1 +``` + +Expected: one path like `/forms/d/e/1FAIpQLS.../formResponse` + +- [ ] **Step 2: Write the spike** + +Create `tools/e1_gate.py`, substituting the four values you just found: + +```python +"""E1: does a cookie-only POST record a response? Spike — hardcoded on purpose. + +Run once. If this does not record a row in F1's response sheet, the whole +direct-POST design is wrong and the plan must be revisited. +""" + +import http.client +import sys +import urllib.parse +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from fastform import cookies # noqa: E402 + +ACTION_PATH = "/forms/d/e/REPLACE_WITH_YOUR_FORM_ID/formResponse" +FIELDS = [ + ("entry.REPLACE_NAME_ID", "E1 Gate Test"), + ("entry.REPLACE_EMAIL_ID", "e1@example.com"), + ("entry.REPLACE_PHONE_ID", "0800000000"), + ("fvv", "1"), + ("pageHistory", "0"), + ("fbzx", "-1234567890123456789"), + ("submissionTimestamp", "-1"), +] + + +def main() -> int: + cookie = cookies.load() + body = urllib.parse.urlencode(FIELDS, encoding="utf-8") + + conn = http.client.HTTPSConnection("docs.google.com", timeout=15) + try: + conn.request( + "POST", + ACTION_PATH, + body=body, + headers={ + "Cookie": cookie, + "User-Agent": cookies.USER_AGENT, + "Content-Type": "application/x-www-form-urlencoded", + "Referer": "https://docs.google.com" + ACTION_PATH.replace( + "/formResponse", "/viewform" + ), + }, + ) + resp = conn.getresponse() + payload = resp.read() + finally: + conn.close() + + print(f"status={resp.status}") + print(f"location={resp.getheader('Location')}") + print(f"bytes={len(payload)}") + Path("fixtures/resp-e1.bin").write_bytes(payload) + print("body written to fixtures/resp-e1.bin") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **Step 3: Run it** + +```bash +cd /Users/supa/projects/active/fastform +python3 tools/e1_gate.py +``` + +Expected: `status=200` and a non-trivial byte count. A `status=302` with a `location` on `accounts.google.com` means the cookie is not being accepted. + +- [ ] **Step 4: Check the response sheet — this is the actual gate** + +Open F1 in Google Forms → **Responses** tab. + +**PASS:** a new row appeared containing `E1 Gate Test`. The design is viable. Continue. + +**FAIL:** no new row. Stop here. Capture what you can and re-plan: + +```bash +head -c 2000 fixtures/resp-e1.bin +``` + +Compare the request in `tools/e1_gate.py` against `fixtures/f1-manual-submit.curl.txt` field by field — the cURL contains something the spike is missing. Common culprits are additional `entry.*_sentinel` fields, a `draftResponse` parameter, or a required header. Do not proceed to Task 4 until a row appears. + +- [ ] **Step 5: Commit** + +```bash +git add tools/e1_gate.py +git commit -m "Add E1 gate spike: cookie-only POST records a response" +``` + +--- + +### Task 4: Create test forms F2, F3, F4 + +**Files:** +- Modify: `fixtures/README.md` +- Create: `fixtures/f3-viewform.html` +- Create: `fixtures/f4-viewform.html` + +**Interfaces:** +- Consumes: nothing in code. +- Produces: the multi-section and validation fixtures Tasks 5 and 9 test against. + +Deferred until now on purpose: if Task 3 had failed, these forms would have been wasted effort. + +- [ ] **Step 1: Create F2** + +Duplicate F1 (Forms → ⋮ → Make a copy), title it `FastForm F2`, and change one setting: **Limit to 1 response: OFF**. Keep sign-in required by leaving "Collect email addresses" set to **Responder input**. + +Save its viewform URL. + +- [ ] **Step 2: Create F3 — multi-section, mixed types** + +Title `FastForm F3`. Sign-in required, limit-to-1 off. Three sections: + +**Section 1:** +- `What is your full name?` — Short answer, Required +- `Tell us about yourself` — Paragraph, Optional + +**Section 2:** +- `Which city?` — Multiple choice, Required. Options: `Bangkok`, `Chiang Mai`, `Phuket`, plus **Other** +- `Pick your sizes` — Checkboxes, Optional. Options: `Small`, `Medium`, `Large` +- `Choose a department` — Dropdown, Required. Options: `Engineering`, `Design`, `Sales` + +**Section 3:** +- `Rate your confidence` — Linear scale 1–5, Required +- `Preferred date` — Date, Optional +- `Preferred time` — Time, Optional + +Save its viewform URL. + +- [ ] **Step 3: Create F4 — validation and a file upload** + +Title `FastForm F4`. Sign-in required. One section: + +- `Numeric code` — Short answer, Required, Response validation → Number → Between → `1000` and `9999` +- `Contact email` — Short answer, Required, Response validation → Text → Email address +- `Upload your entry` — **File upload**, Optional + +Save its viewform URL. + +- [ ] **Step 4: Capture F3 and F4 page fixtures** + +```bash +cd /Users/supa/projects/active/fastform +curl -s -H "Cookie: $(cat config/cookies.txt)" '' > fixtures/f3-viewform.html +curl -s -H "Cookie: $(cat config/cookies.txt)" '' > fixtures/f4-viewform.html +grep -c FB_PUBLIC_LOAD_DATA_ fixtures/f3-viewform.html fixtures/f4-viewform.html +``` + +Expected: `fixtures/f3-viewform.html:1` and `fixtures/f4-viewform.html:1` + +- [ ] **Step 5: Record the URLs** + +Append to `fixtures/README.md`: + +```markdown + +## Test forms + +| Form | Purpose | URL | +|------|---------|-----| +| F1 | limit-to-1 ON, 3 required short answers | | +| F2 | limit-to-1 OFF, otherwise identical to F1 | | +| F3 | 3 sections, mixed question types | | +| F4 | validation rules + file upload | | + +- `f3-viewform.html` — multi-section fixture for parse tests. +- `f4-viewform.html` — validation and file-upload fixture for blocker detection. +``` + +- [ ] **Step 6: Commit** + +```bash +git add fixtures/README.md fixtures/f1-viewform.html fixtures/f3-viewform.html fixtures/f4-viewform.html +git commit -m "Add test forms F2-F4 and page fixtures" +``` + +--- + +### Task 5: Parse form structure + +**Files:** +- Create: `fastform/parse.py` +- Create: `tests/test_parse.py` + +**Interfaces:** +- Consumes: fixtures from Tasks 1 and 4. +- Produces: + - `Question` dataclass: `entry_id: str`, `text: str`, `type: int`, `required: bool`, `options: list[str]` + - `FormSpec` dataclass: `form_id: str`, `action_path: str`, `page_count: int`, `questions: list[Question]`; properties `page_history: str`, `blockers: list[Question]` + - `parse(html: bytes, url: str) -> FormSpec` + - `form_id_from_url(url: str) -> str` + - `TYPE_FILE_UPLOAD`, `TYPE_SECTION`, and the other type constants. + +**Important:** the index positions inside `FB_PUBLIC_LOAD_DATA_` below are my best understanding, not verified fact. Step 1 makes you check them against your own fixture before writing code. If they differ, adjust the constants — the tests assert on question text and entry IDs you control, so they will tell you when you have it right. + +- [ ] **Step 1: Inspect the real structure first** + +```bash +cd /Users/supa/projects/active/fastform +python3 - <<'PY' +import json, re +html = open("fixtures/f1-viewform.html", "rb").read() +m = re.search(rb"FB_PUBLIC_LOAD_DATA_\s*=\s*(\[.*?\]);\s*", html, re.S) +print("matched:", bool(m)) +data = json.loads(m.group(1)) +print("top-level length:", len(data)) +items = data[1][1] +print("item count:", len(items)) +for it in items: + print(json.dumps(it)[:220]) +PY +``` + +Expected: three items printed, each containing one of your F1 question texts, and each containing a nine- or ten-digit number that matches an `entry.*` ID from Task 3 Step 1. + +Note for each item which position holds the title, which holds the type, and where the entry ID and required flag sit. The code below assumes `item[1]` = title, `item[3]` = type, `item[4][0][0]` = entry id, `item[4][0][1]` = options, `item[4][0][2]` = required. + +- [ ] **Step 2: Write the failing test** + +Create `tests/test_parse.py`: + +```python +import unittest +from pathlib import Path + +from fastform import parse + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" + +F1_URL = "https://docs.google.com/forms/d/e/FAKEID123/viewform" + + +class TestFormIdFromUrl(unittest.TestCase): + def test_extracts_published_id(self): + self.assertEqual( + parse.form_id_from_url( + "https://docs.google.com/forms/d/e/1FAIpQLSabc/viewform?usp=sf_link" + ), + "1FAIpQLSabc", + ) + + def test_rejects_non_form_url(self): + with self.assertRaises(ValueError): + parse.form_id_from_url("https://example.com/hello") + + +class TestParseF1(unittest.TestCase): + @classmethod + def setUpClass(cls): + html = (FIXTURES / "f1-viewform.html").read_bytes() + cls.spec = parse.parse(html, F1_URL) + + def test_finds_three_questions(self): + self.assertEqual(len(self.spec.questions), 3) + + def test_question_text_is_readable(self): + texts = [q.text for q in self.spec.questions] + self.assertIn("What is your full name?", texts) + + def test_entry_ids_are_numeric_strings(self): + for q in self.spec.questions: + self.assertTrue(q.entry_id.isdigit(), q.entry_id) + + def test_all_required(self): + self.assertTrue(all(q.required for q in self.spec.questions)) + + def test_single_page(self): + self.assertEqual(self.spec.page_count, 1) + self.assertEqual(self.spec.page_history, "0") + + def test_action_path(self): + self.assertEqual( + self.spec.action_path, "/forms/d/e/FAKEID123/formResponse" + ) + + def test_no_blockers(self): + self.assertEqual(self.spec.blockers, []) + + +class TestParseF3(unittest.TestCase): + @classmethod + def setUpClass(cls): + html = (FIXTURES / "f3-viewform.html").read_bytes() + cls.spec = parse.parse(html, F1_URL) + + def test_three_pages(self): + self.assertEqual(self.spec.page_count, 3) + self.assertEqual(self.spec.page_history, "0,1,2") + + def test_choice_options_captured(self): + city = next(q for q in self.spec.questions if "city" in q.text.lower()) + self.assertIn("Bangkok", city.options) + self.assertIn("Chiang Mai", city.options) + + def test_optional_question_detected(self): + optional = [q for q in self.spec.questions if not q.required] + self.assertTrue(optional, "expected at least one optional question") + + +class TestParseF4(unittest.TestCase): + @classmethod + def setUpClass(cls): + html = (FIXTURES / "f4-viewform.html").read_bytes() + cls.spec = parse.parse(html, F1_URL) + + def test_file_upload_is_a_blocker(self): + self.assertEqual(len(self.spec.blockers), 1) + self.assertEqual(self.spec.blockers[0].type, parse.TYPE_FILE_UPLOAD) + + +class TestParseFailures(unittest.TestCase): + def test_missing_load_data_raises(self): + with self.assertRaises(parse.NotParseable): + parse.parse(b"form is closed", F1_URL) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 3: Run the test to verify it fails** + +```bash +python3 -m unittest tests.test_parse -v +``` + +Expected: `ModuleNotFoundError: No module named 'fastform.parse'` + +- [ ] **Step 4: Write the implementation** + +Create `fastform/parse.py`: + +```python +"""Turn a Google Form page into a FormSpec. + +Everything comes out of the FB_PUBLIC_LOAD_DATA_ blob embedded in the page. +A closed form does not include that blob, which is why NotParseable exists and +why the caller has to treat it as a distinct state rather than an error. +""" + +import json +import re +from dataclasses import dataclass, field + +TYPE_SHORT_ANSWER = 0 +TYPE_PARAGRAPH = 1 +TYPE_MULTIPLE_CHOICE = 2 +TYPE_DROPDOWN = 3 +TYPE_CHECKBOXES = 4 +TYPE_LINEAR_SCALE = 5 +TYPE_GRID = 7 +TYPE_SECTION = 8 +TYPE_DATE = 9 +TYPE_TIME = 10 +TYPE_FILE_UPLOAD = 13 + +#: Types the direct-POST path cannot satisfy. Hitting one means browser handoff. +BLOCKER_TYPES = frozenset({TYPE_FILE_UPLOAD, TYPE_GRID}) + +#: Types whose value must match one of the offered options. +CHOICE_TYPES = frozenset({TYPE_MULTIPLE_CHOICE, TYPE_DROPDOWN, TYPE_CHECKBOXES}) + +_LOAD_DATA_RE = re.compile( + rb"FB_PUBLIC_LOAD_DATA_\s*=\s*(\[.*?\]);\s*", re.S +) +_FORM_ID_RE = re.compile(r"/forms/d/e/([^/]+)/") + + +class NotParseable(Exception): + """The page carries no form structure — closed, deleted, or not a form.""" + + +@dataclass +class Question: + entry_id: str + text: str + type: int + required: bool + options: list[str] = field(default_factory=list) + + +@dataclass +class FormSpec: + form_id: str + action_path: str + page_count: int + questions: list[Question] + + @property + def page_history(self) -> str: + """The pageHistory value that satisfies every section in one POST.""" + return ",".join(str(i) for i in range(self.page_count)) + + @property + def blockers(self) -> list[Question]: + return [q for q in self.questions if q.type in BLOCKER_TYPES] + + +def form_id_from_url(url: str) -> str: + match = _FORM_ID_RE.search(url) + if not match: + raise ValueError(f"not a published Google Form URL: {url}") + return match.group(1) + + +def extract_load_data(html: bytes) -> list: + match = _LOAD_DATA_RE.search(html) + if not match: + raise NotParseable("FB_PUBLIC_LOAD_DATA_ not found") + return json.loads(match.group(1)) + + +def _items(load_data: list) -> list: + try: + return load_data[1][1] or [] + except (IndexError, TypeError) as exc: + raise NotParseable(f"unexpected load data shape: {exc}") from exc + + +def _options(raw_options) -> list[str]: + out = [] + for opt in raw_options or []: + if opt and opt[0]: + out.append(str(opt[0])) + return out + + +def parse(html: bytes, url: str) -> FormSpec: + load_data = extract_load_data(html) + form_id = form_id_from_url(url) + + questions: list[Question] = [] + page_count = 1 + + for item in _items(load_data): + item_type = item[3] + if item_type == TYPE_SECTION: + page_count += 1 + continue + + title = item[1] or "" + for entry in item[4] or []: + questions.append( + Question( + entry_id=str(entry[0]), + text=title, + type=item_type, + required=bool(entry[2]), + options=_options(entry[1]), + ) + ) + + return FormSpec( + form_id=form_id, + action_path=f"/forms/d/e/{form_id}/formResponse", + page_count=page_count, + questions=questions, + ) +``` + +- [ ] **Step 5: Run the test to verify it passes** + +```bash +python3 -m unittest tests.test_parse -v +``` + +Expected: `OK`. + +If index assumptions were wrong you will see failures naming exactly what broke — a question count of 0 means `item[3]`/`item[4]` are misplaced; garbled `text` means `item[1]` is not the title. Adjust the constants in `parse()` and re-run. + +- [ ] **Step 6: Commit** + +```bash +git add fastform/parse.py tests/test_parse.py +git commit -m "Add form structure parser" +``` + +--- + +### Task 6: Match answers to questions + +**Files:** +- Create: `fastform/match.py` +- Create: `tests/test_match.py` + +**Interfaces:** +- Consumes: `Question`, `FormSpec`, `CHOICE_TYPES` from `fastform.parse`. +- Produces: + - `FilledForm` dataclass: `pairs: list[tuple[str, str]]`, `gaps: list[Question]`, `blockers: list[Question]`, `confidence: float` + - `load_bank(path: Path | None = None) -> list[dict]` + - `normalize(text: str) -> str` + - `score_question(question, entry) -> float` + - `fill(spec: FormSpec, bank: list[dict]) -> FilledForm` + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_match.py`: + +```python +import unittest + +from fastform import match +from fastform.parse import ( + TYPE_FILE_UPLOAD, + TYPE_MULTIPLE_CHOICE, + TYPE_SHORT_ANSWER, + FormSpec, + Question, +) + + +def spec(*questions): + return FormSpec( + form_id="X", action_path="/p", page_count=1, questions=list(questions) + ) + + +BANK = [ + {"match": ["full name", "your name"], "value": "Test Person"}, + {"match": ["email address"], "regex": "e-?mail", "value": "test@example.com"}, + {"match": ["which city"], "value": "Bangkok"}, +] + + +class TestNormalize(unittest.TestCase): + def test_casefolds_and_strips_punctuation(self): + self.assertEqual(match.normalize("What is your Name?"), "what is your name") + + def test_collapses_whitespace(self): + self.assertEqual(match.normalize("a b\n c"), "a b c") + + +class TestFill(unittest.TestCase): + def test_exact_text_match_fills(self): + s = spec(Question("111", "full name", TYPE_SHORT_ANSWER, True)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, [("111", "Test Person")]) + self.assertEqual(filled.gaps, []) + + def test_regex_wins_and_scores_one(self): + s = spec(Question("222", "Your E-Mail", TYPE_SHORT_ANSWER, True)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, [("222", "test@example.com")]) + self.assertEqual(filled.confidence, 1.0) + + def test_unmatched_required_becomes_a_gap(self): + s = spec(Question("333", "Favourite dinosaur", TYPE_SHORT_ANSWER, True)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, []) + self.assertEqual([q.entry_id for q in filled.gaps], ["333"]) + + def test_unmatched_optional_is_omitted_not_a_gap(self): + s = spec(Question("444", "Favourite dinosaur", TYPE_SHORT_ANSWER, False)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, []) + self.assertEqual(filled.gaps, []) + + def test_choice_value_must_exist_in_options(self): + s = spec( + Question( + "555", + "Which city?", + TYPE_MULTIPLE_CHOICE, + True, + options=["Bangkok Metropolitan Region", "Phuket"], + ) + ) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, []) + self.assertEqual([q.entry_id for q in filled.gaps], ["555"]) + + def test_choice_value_present_in_options_fills(self): + s = spec( + Question( + "666", + "Which city?", + TYPE_MULTIPLE_CHOICE, + True, + options=["Bangkok", "Phuket"], + ) + ) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, [("666", "Bangkok")]) + + def test_blockers_are_reported_separately(self): + s = spec(Question("777", "Upload", TYPE_FILE_UPLOAD, False)) + filled = match.fill(s, BANK) + self.assertEqual([q.entry_id for q in filled.blockers], ["777"]) + + def test_confidence_is_min_over_required_questions(self): + s = spec( + Question("111", "full name", TYPE_SHORT_ANSWER, True), + Question("333", "Favourite dinosaur", TYPE_SHORT_ANSWER, True), + ) + filled = match.fill(s, BANK) + self.assertEqual(filled.confidence, 0.0) + + def test_confidence_is_one_when_no_required_questions(self): + s = spec(Question("444", "Anything", TYPE_SHORT_ANSWER, False)) + self.assertEqual(match.fill(s, BANK).confidence, 1.0) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +python3 -m unittest tests.test_match -v +``` + +Expected: `ModuleNotFoundError: No module named 'fastform.match'` + +- [ ] **Step 3: Write the implementation** + +Create `fastform/match.py`: + +```python +"""Match prepared answers to a form's questions by text, never by position. + +Position matching breaks the moment a question is added or reordered. Text +matching degrades gracefully: a near miss still scores, and the score is what +decides whether firing is safe or the operator has to look. +""" + +import difflib +import re +import tomllib +import unicodedata +from dataclasses import dataclass, field +from pathlib import Path + +from fastform.parse import BLOCKER_TYPES, CHOICE_TYPES, FormSpec, Question + +DEFAULT_BANK_PATH = Path(__file__).resolve().parent.parent / "config" / "answers.toml" + +#: Below this, a fuzzy hit is treated as no match at all. +FUZZY_FLOOR = 0.6 + +_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE) + + +@dataclass +class FilledForm: + pairs: list[tuple[str, str]] = field(default_factory=list) + gaps: list[Question] = field(default_factory=list) + blockers: list[Question] = field(default_factory=list) + confidence: float = 1.0 + + +def load_bank(path: Path | None = None) -> list[dict]: + path = path or DEFAULT_BANK_PATH + with path.open("rb") as fh: + return tomllib.load(fh).get("answer", []) + + +def normalize(text: str) -> str: + text = unicodedata.normalize("NFKC", text).casefold() + text = _PUNCT_RE.sub(" ", text) + return " ".join(text.split()) + + +def score_question(question: Question, entry: dict) -> float: + """How well does this bank entry match this question? 0.0 means no match.""" + pattern = entry.get("regex") + if pattern and re.search(pattern, question.text, re.IGNORECASE): + return 1.0 + + target = normalize(question.text) + target_tokens = set(target.split()) + best = 0.0 + + for phrase in entry.get("match", []): + candidate = normalize(phrase) + if not candidate: + continue + if candidate == target: + best = max(best, 0.95) + continue + if set(candidate.split()) <= target_tokens: + best = max(best, 0.8) + continue + ratio = difflib.SequenceMatcher(None, candidate, target).ratio() + if ratio >= FUZZY_FLOOR: + best = max(best, ratio) + + return best + + +def _resolve_choice(value: str, options: list[str]) -> str | None: + """A choice answer must be one of the offered options, or it will be rejected.""" + if not options: + return value + normalized = {normalize(o): o for o in options} + return normalized.get(normalize(value)) + + +def fill(spec: FormSpec, bank: list[dict]) -> FilledForm: + filled = FilledForm() + scores: list[float] = [] + + for question in spec.questions: + if question.type in BLOCKER_TYPES: + filled.blockers.append(question) + continue + + best_score, best_value = 0.0, None + for entry in bank: + score = score_question(question, entry) + if score > best_score: + best_score, best_value = score, entry["value"] + + if best_value is not None and question.type in CHOICE_TYPES: + best_value = _resolve_choice(best_value, question.options) + if best_value is None: + best_score = 0.0 + + if best_value is not None and best_score > 0.0: + filled.pairs.append((question.entry_id, best_value)) + if question.required: + scores.append(best_score) + elif question.required: + filled.gaps.append(question) + scores.append(0.0) + + filled.confidence = min(scores) if scores else 1.0 + return filled +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +python3 -m unittest tests.test_match -v +``` + +Expected: `Ran 12 tests` … `OK` + +- [ ] **Step 5: Commit** + +```bash +git add fastform/match.py tests/test_match.py +git commit -m "Add text-based answer matching with confidence scoring" +``` + +--- + +### Task 7: Compile the request + +**Files:** +- Create: `fastform/compile.py` +- Create: `tests/test_compile.py` + +**Interfaces:** +- Consumes: `FormSpec` from `fastform.parse`, `FilledForm` from `fastform.match`, `USER_AGENT` from `fastform.cookies`. +- Produces: + - `RequestBlob` dataclass: `host: str`, `path: str`, `body: str`, `headers: dict[str, str]` + - `new_fbzx() -> str` + - `build(spec, filled, cookie, fbzx=None) -> RequestBlob` + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_compile.py`: + +```python +import unittest +import urllib.parse + +from fastform import compile as compile_mod +from fastform.match import FilledForm +from fastform.parse import FormSpec + + +def spec(pages=1): + return FormSpec( + form_id="ABC", action_path="/forms/d/e/ABC/formResponse", + page_count=pages, questions=[], + ) + + +class TestNewFbzx(unittest.TestCase): + def test_is_a_signed_integer_string(self): + value = compile_mod.new_fbzx() + int(value) + + def test_differs_between_calls(self): + self.assertNotEqual(compile_mod.new_fbzx(), compile_mod.new_fbzx()) + + +class TestBuild(unittest.TestCase): + def _fields(self, blob): + return dict(urllib.parse.parse_qsl(blob.body)) + + def test_entry_pairs_are_prefixed(self): + filled = FilledForm(pairs=[("123", "Alice")]) + blob = compile_mod.build(spec(), filled, "SID=x", fbzx="-1") + self.assertEqual(self._fields(blob)["entry.123"], "Alice") + + def test_includes_fvv_and_timestamp(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + fields = self._fields(blob) + self.assertEqual(fields["fvv"], "1") + self.assertEqual(fields["submissionTimestamp"], "-1") + + def test_page_history_covers_every_section(self): + blob = compile_mod.build(spec(pages=3), FilledForm(), "SID=x", fbzx="-1") + self.assertEqual(self._fields(blob)["pageHistory"], "0,1,2") + + def test_cookie_and_content_type_headers(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=abc", fbzx="-1") + self.assertEqual(blob.headers["Cookie"], "SID=abc") + self.assertEqual( + blob.headers["Content-Type"], "application/x-www-form-urlencoded" + ) + + def test_referer_points_at_the_viewform(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + self.assertEqual( + blob.headers["Referer"], + "https://docs.google.com/forms/d/e/ABC/viewform", + ) + + def test_unicode_values_survive_encoding(self): + filled = FilledForm(pairs=[("1", "กรุงเทพ")]) + blob = compile_mod.build(spec(), filled, "SID=x", fbzx="-1") + self.assertEqual(self._fields(blob)["entry.1"], "กรุงเทพ") + + def test_generates_fbzx_when_not_supplied(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x") + int(self._fields(blob)["fbzx"]) + + def test_content_length_matches_encoded_body(self): + filled = FilledForm(pairs=[("1", "กรุงเทพ")]) + blob = compile_mod.build(spec(), filled, "SID=x", fbzx="-1") + self.assertEqual( + blob.headers["Content-Length"], + str(len(blob.body.encode("utf-8"))), + ) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +python3 -m unittest tests.test_compile -v +``` + +Expected: `ModuleNotFoundError: No module named 'fastform.compile'` + +- [ ] **Step 3: Write the implementation** + +Create `fastform/compile.py`: + +```python +"""Turn a filled form into a ready-to-send request. + +Everything expensive happens here, at arm time. Sending is then a socket write +with nothing left to compute — which is the whole point of separating this from +transport. +""" + +import random +import urllib.parse +from dataclasses import dataclass + +from fastform.cookies import USER_AGENT +from fastform.match import FilledForm +from fastform.parse import FormSpec + +HOST = "docs.google.com" + + +@dataclass +class RequestBlob: + host: str + path: str + body: str + headers: dict[str, str] + + +def new_fbzx() -> str: + """A fresh anti-duplicate token, in the same shape Google's own pages emit.""" + return str(random.randint(-(2**63), 2**63 - 1)) + + +def build( + spec: FormSpec, + filled: FilledForm, + cookie: str, + fbzx: str | None = None, +) -> RequestBlob: + fields: list[tuple[str, str]] = [ + (f"entry.{entry_id}", value) for entry_id, value in filled.pairs + ] + fields.append(("fvv", "1")) + fields.append(("fbzx", fbzx or new_fbzx())) + fields.append(("pageHistory", spec.page_history)) + fields.append(("submissionTimestamp", "-1")) + + body = urllib.parse.urlencode(fields, encoding="utf-8") + viewform = f"https://{HOST}/forms/d/e/{spec.form_id}/viewform" + + return RequestBlob( + host=HOST, + path=spec.action_path, + body=body, + headers={ + "Cookie": cookie, + "User-Agent": USER_AGENT, + "Content-Type": "application/x-www-form-urlencoded", + "Content-Length": str(len(body.encode("utf-8"))), + "Referer": viewform, + "Origin": f"https://{HOST}", + }, + ) +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +python3 -m unittest tests.test_compile -v +``` + +Expected: `Ran 10 tests` … `OK` + +- [ ] **Step 5: Commit** + +```bash +git add fastform/compile.py tests/test_compile.py +git commit -m "Add request compiler" +``` + +--- + +### Task 8: Dry-run tool and the cURL diff + +**Files:** +- Create: `tools/dryrun.py` +- Create: `tools/curl_diff.py` + +**Interfaces:** +- Consumes: `cookies`, `parse`, `match`, `compile` from Tasks 2, 5, 6, 7. +- Produces: `tools/dryrun.py --url ` printing the request that would be sent. No later task imports these. + +This is the highest-value verification in the plan: it catches every missing field at once by comparing against a request Google actually accepted. + +- [ ] **Step 1: Write the dry-run tool** + +Create `tools/dryrun.py`: + +```python +"""Build the request for a form and print it. Sends nothing. + +Usage: + python3 tools/dryrun.py --url +""" + +import argparse +import http.client +import sys +import urllib.parse +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from fastform import compile as compile_mod # noqa: E402 +from fastform import cookies, match, parse # noqa: E402 + + +def fetch(url: str, cookie: str) -> bytes: + parts = urllib.parse.urlsplit(url) + conn = http.client.HTTPSConnection(parts.netloc, timeout=15) + try: + conn.request( + "GET", + parts.path + ("?" + parts.query if parts.query else ""), + headers={"Cookie": cookie, "User-Agent": cookies.USER_AGENT}, + ) + resp = conn.getresponse() + return resp.read() + finally: + conn.close() + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--url", required=True) + args = ap.parse_args() + + cookie = cookies.load() + html = fetch(args.url, cookie) + + try: + spec = parse.parse(html, args.url) + except parse.NotParseable as exc: + print(f"NOT PARSEABLE: {exc}") + print("The form is closed, deleted, or the cookie is not signed in.") + return 2 + + filled = match.fill(spec, match.load_bank()) + blob = compile_mod.build(spec, filled, cookie) + + print(f"form_id {spec.form_id}") + print(f"pages {spec.page_count} (pageHistory={spec.page_history})") + print(f"questions {len(spec.questions)}") + print(f"confidence {filled.confidence:.2f}") + print() + + for question in spec.questions: + value = dict(filled.pairs).get(question.entry_id) + flag = "REQ" if question.required else "opt" + status = value if value is not None else "— unfilled —" + print(f" [{flag}] entry.{question.entry_id:<12} {question.text[:44]:<46} {status}") + + if filled.gaps: + print(f"\nGAPS ({len(filled.gaps)}) — required and unmatched:") + for question in filled.gaps: + print(f" entry.{question.entry_id} {question.text}") + + if filled.blockers: + print(f"\nBLOCKERS ({len(filled.blockers)}) — direct POST cannot satisfy these:") + for question in filled.blockers: + print(f" entry.{question.entry_id} type={question.type} {question.text}") + + print(f"\nPOST https://{blob.host}{blob.path}") + for key, value in blob.headers.items(): + shown = "" if key == "Cookie" else value + print(f" {key}: {shown}") + print(f"\nbody: {blob.body}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **Step 2: Run it against F1** + +```bash +cd /Users/supa/projects/active/fastform +python3 tools/dryrun.py --url '' +``` + +Expected: three questions listed, all filled, `confidence 0.80` or higher, no gaps, no blockers, and a body containing three `entry.*` fields plus `fvv`, `fbzx`, `pageHistory`, `submissionTimestamp`. + +- [ ] **Step 3: Run it against F4 to confirm blocker detection** + +```bash +python3 tools/dryrun.py --url '' +``` + +Expected: a `BLOCKERS (1)` section naming the file upload question with `type=13`. + +- [ ] **Step 4: Write the cURL diff tool** + +Create `tools/curl_diff.py`: + +```python +"""Compare our generated POST body against a real one captured from DevTools. + +Any field Google's own page sent that we do not is a candidate explanation for +a rejected submission. + +Usage: + python3 tools/curl_diff.py --curl fixtures/f1-manual-submit.curl.txt --url +""" + +import argparse +import re +import sys +import urllib.parse +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from fastform import compile as compile_mod # noqa: E402 +from fastform import cookies, match, parse # noqa: E402 +from tools.dryrun import fetch # noqa: E402 + +_DATA_RE = re.compile(r"--data(?:-raw|-urlencode)?\s+\$?'(.*?)'", re.S) + + +def curl_fields(text: str) -> dict[str, str]: + chunks = _DATA_RE.findall(text) + if not chunks: + raise SystemExit("no --data section found in the cURL file") + return dict(urllib.parse.parse_qsl("&".join(chunks), keep_blank_values=True)) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--curl", required=True) + ap.add_argument("--url", required=True) + args = ap.parse_args() + + theirs = curl_fields(Path(args.curl).read_text(encoding="utf-8")) + + cookie = cookies.load() + spec = parse.parse(fetch(args.url, cookie), args.url) + filled = match.fill(spec, match.load_bank()) + ours = dict( + urllib.parse.parse_qsl( + compile_mod.build(spec, filled, cookie).body, keep_blank_values=True + ) + ) + + missing = sorted(set(theirs) - set(ours)) + extra = sorted(set(ours) - set(theirs)) + + print(f"chrome sent {len(theirs)} fields, we send {len(ours)}\n") + + if missing: + print("MISSING — Chrome sent these and we do not:") + for key in missing: + print(f" {key} = {theirs[key]!r}") + else: + print("MISSING — none") + + print() + if extra: + print("EXTRA — we send these and Chrome did not:") + for key in extra: + print(f" {key} = {ours[key]!r}") + else: + print("EXTRA — none") + + return 1 if missing else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **Step 5: Run the diff** + +```bash +touch tools/__init__.py +python3 tools/curl_diff.py --curl fixtures/f1-manual-submit.curl.txt --url '' +``` + +Expected: `MISSING — none`. + +If fields are listed as missing, add them to `fastform/compile.py`'s `fields` list, add a test in `tests/test_compile.py` asserting each new field is present, and re-run until the diff is clean. Common additions are `entry._sentinel` fields for checkbox questions and a `draftResponse` parameter. + +- [ ] **Step 6: Commit** + +```bash +git add tools/dryrun.py tools/curl_diff.py tools/__init__.py +git commit -m "Add dry-run and cURL diff verification tools" +``` + +--- + +### Task 9: Classify responses + +**Files:** +- Create: `fastform/classify.py` +- Create: `tests/test_classify.py` +- Create: `tools/capture_responses.py` + +**Interfaces:** +- Consumes: `cookies`, `parse`, `match`, `compile` from earlier tasks. +- Produces: + - `Outcome` enum: `RECORDED`, `REJECTED`, `CLOSED`, `ALREADY_RESPONDED`, `AUTH_FAIL`, `UNKNOWN` + - `classify(status: int, location: str | None, body: bytes) -> Outcome` + +The fixtures are the specification here. Capture them first, look at them, then make the classifier satisfy them. + +- [ ] **Step 1: Write the capture tool** + +Create `tools/capture_responses.py`: + +```python +"""Capture real response bodies for each outcome, to use as classifier fixtures. + +Usage: + python3 tools/capture_responses.py --url --name success +""" + +import argparse +import http.client +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from fastform import compile as compile_mod # noqa: E402 +from fastform import cookies, match, parse # noqa: E402 +from tools.dryrun import fetch # noqa: E402 + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--url", required=True) + ap.add_argument("--name", required=True, help="fixture suffix, e.g. success") + ap.add_argument( + "--drop-required", + action="store_true", + help="omit every required field, to provoke a validation rejection", + ) + args = ap.parse_args() + + cookie = cookies.load() + html = fetch(args.url, cookie) + + try: + spec = parse.parse(html, args.url) + except parse.NotParseable: + FIXTURES.joinpath(f"resp-{args.name}.bin").write_bytes(html) + print(f"form not parseable; saved the GET body as resp-{args.name}.bin") + return 0 + + filled = match.fill(spec, match.load_bank()) + if args.drop_required: + filled.pairs = [] + + blob = compile_mod.build(spec, filled, cookie) + conn = http.client.HTTPSConnection(blob.host, timeout=15) + try: + conn.request("POST", blob.path, body=blob.body, headers=blob.headers) + resp = conn.getresponse() + body = resp.read() + finally: + conn.close() + + out = FIXTURES / f"resp-{args.name}.bin" + out.write_bytes(body) + print(f"status={resp.status} location={resp.getheader('Location')} bytes={len(body)}") + print(f"saved {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **Step 2: Capture all five fixtures** + +```bash +cd /Users/supa/projects/active/fastform + +# 1. success — F2 has limit-to-1 off, so this is repeatable +python3 tools/capture_responses.py --url '' --name success + +# 2. rejected — same form, required fields stripped +python3 tools/capture_responses.py --url '' --name rejected --drop-required + +# 3. already-responded — F1 has limit-to-1 on and already has your E1 row +python3 tools/capture_responses.py --url '' --name already + +# 4. closed — turn OFF "Accepting responses" on F2 in the Forms UI first, then: +python3 tools/capture_responses.py --url '' --name closed +# then turn accepting responses back ON + +# 5. auth-fail +printf 'SID=definitely-not-valid' > /tmp/bad-cookies.txt +cp config/cookies.txt /tmp/real-cookies.txt +cp /tmp/bad-cookies.txt config/cookies.txt +python3 tools/capture_responses.py --url '' --name authfail +cp /tmp/real-cookies.txt config/cookies.txt +``` + +Verify all five exist: + +```bash +ls -la fixtures/resp-*.bin +``` + +Expected: `resp-success.bin`, `resp-rejected.bin`, `resp-already.bin`, `resp-closed.bin`, `resp-authfail.bin` + +- [ ] **Step 3: Inspect what actually distinguishes them** + +```bash +for f in fixtures/resp-*.bin; do + echo "=== $f ($(wc -c < "$f") bytes)" + echo -n " FB_PUBLIC_LOAD_DATA_: "; grep -c FB_PUBLIC_LOAD_DATA_ "$f" || true + echo -n " freebirdFormviewerViewResponseConfirm: "; grep -c freebirdFormviewerViewResponseConfirm "$f" || true +done +``` + +Read the output. Note which markers are present in which fixture — the implementation in Step 5 uses these, and **your values may differ from the ones written there.** Adjust the constants to match what you observe. + +- [ ] **Step 4: Write the failing test** + +Create `tests/test_classify.py`: + +```python +import unittest +from pathlib import Path + +from fastform.classify import Outcome, classify + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" + +# Each fixture was captured from a real submission in Task 9 Step 2. +CASES = [ + ("resp-success.bin", 200, None, Outcome.RECORDED), + ("resp-rejected.bin", 200, None, Outcome.REJECTED), + ("resp-already.bin", 200, None, Outcome.ALREADY_RESPONDED), + ("resp-closed.bin", 200, None, Outcome.CLOSED), +] + + +class TestClassifyFixtures(unittest.TestCase): + def test_each_fixture_maps_to_its_outcome(self): + for name, status, location, expected in CASES: + with self.subTest(fixture=name): + body = (FIXTURES / name).read_bytes() + self.assertEqual(classify(status, location, body), expected) + + +class TestClassifyStructural(unittest.TestCase): + def test_redirect_to_accounts_is_auth_fail(self): + self.assertEqual( + classify(302, "https://accounts.google.com/signin", b""), + Outcome.AUTH_FAIL, + ) + + def test_server_error_is_unknown(self): + self.assertEqual(classify(500, None, b"oops"), Outcome.UNKNOWN) + + def test_empty_body_is_unknown(self): + self.assertEqual(classify(200, None, b""), Outcome.UNKNOWN) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 5: Write the implementation** + +Create `fastform/classify.py`: + +```python +"""Decide what a response means. + +Structural signals first, language-dependent strings only as a fallback: these +pages render in the account's UI language, so an English marker is not something +to rely on. The markers below were derived from real fixtures — see +tests/test_classify.py. +""" + +from enum import Enum + +#: Present whenever a live form is rendered. Its reappearance after a POST means +#: the submission bounced and we are looking at the form again. +FORM_DATA_MARKER = b"FB_PUBLIC_LOAD_DATA_" + +#: Present on the "your response has been recorded" page. +CONFIRM_MARKERS = ( + b"freebirdFormviewerViewResponseConfirmationMessage", + b"freebirdFormviewerViewResponseConfirmContentContainer", +) + +#: Present when the form exists but is not accepting responses. +CLOSED_MARKERS = (b"no longer accepting responses",) + +#: Present when limit-to-one has already been satisfied by this account. +ALREADY_MARKERS = ( + b"freebirdFormviewerViewResponsePreviouslyRespondedText", + b"already responded", +) + + +class Outcome(str, Enum): + RECORDED = "RECORDED" + REJECTED = "REJECTED" + CLOSED = "CLOSED" + ALREADY_RESPONDED = "ALREADY_RESPONDED" + AUTH_FAIL = "AUTH_FAIL" + UNKNOWN = "UNKNOWN" + + +def _has_any(body: bytes, markers) -> bool: + lowered = body.lower() + return any(marker.lower() in lowered for marker in markers) + + +def classify(status: int, location: str | None, body: bytes) -> Outcome: + if status in (301, 302, 303, 307, 308): + if location and "accounts.google.com" in location: + return Outcome.AUTH_FAIL + return Outcome.UNKNOWN + + if status != 200 or not body: + return Outcome.UNKNOWN + + # Order matters. Confirmation is checked before "closed" because a success + # page also lacks FB_PUBLIC_LOAD_DATA_, and a loose closed-marker would + # otherwise swallow it. + if _has_any(body, ALREADY_MARKERS): + return Outcome.ALREADY_RESPONDED + if _has_any(body, CONFIRM_MARKERS): + return Outcome.RECORDED + if _has_any(body, CLOSED_MARKERS): + return Outcome.CLOSED + if FORM_DATA_MARKER in body: + return Outcome.REJECTED + + return Outcome.UNKNOWN +``` + +- [ ] **Step 6: Run the test** + +```bash +python3 -m unittest tests.test_classify -v +``` + +Expected: `OK`. + +If a fixture maps to the wrong outcome, the markers above do not match your account's pages. Go back to Step 3's output, find a string that is present in exactly one fixture, and use it. **Do not weaken the test to match the code** — the fixtures are ground truth. + +- [ ] **Step 7: Commit** + +```bash +git add fastform/classify.py tests/test_classify.py tools/capture_responses.py fixtures/resp-*.bin +git commit -m "Add response classifier with captured fixtures" +``` + +--- + +### Task 10: Warm connection transport + +**Files:** +- Create: `fastform/transport.py` +- Create: `tests/test_transport.py` + +**Interfaces:** +- Consumes: `USER_AGENT` from `fastform.cookies`, `RequestBlob` from `fastform.compile`. +- Produces: + - `Response` dataclass: `status: int`, `location: str | None`, `body: bytes`, `elapsed_ms: float` + - `WarmConnection` class: `connect()`, `heartbeat()`, `get(path, cookie) -> Response`, `send(blob) -> Response`, `close()` + +- [ ] **Step 1: Write the failing test** + +Uses a local `http.server`, so no external network. + +Create `tests/test_transport.py`: + +```python +import http.server +import threading +import unittest + +from fastform.compile import RequestBlob +from fastform.transport import Response, WarmConnection + + +class _Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b"echo:" + body) + + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, *args): + pass + + +class TestWarmConnection(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.server = http.server.HTTPServer(("127.0.0.1", 0), _Handler) + cls.port = cls.server.server_address[1] + cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) + cls.thread.start() + + @classmethod + def tearDownClass(cls): + cls.server.shutdown() + cls.server.server_close() + + def _conn(self): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False) + conn.connect() + self.addCleanup(conn.close) + return conn + + def test_send_returns_body_and_status(self): + conn = self._conn() + blob = RequestBlob( + host="127.0.0.1", path="/x", body="a=1", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + resp = conn.send(blob) + self.assertIsInstance(resp, Response) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.body, b"echo:a=1") + + def test_records_elapsed_time(self): + conn = self._conn() + blob = RequestBlob(host="127.0.0.1", path="/x", body="a=1", headers={}) + self.assertGreater(conn.send(blob).elapsed_ms, 0.0) + + def test_connection_is_reusable(self): + conn = self._conn() + blob = RequestBlob(host="127.0.0.1", path="/x", body="a=1", headers={}) + self.assertEqual(conn.send(blob).status, 200) + self.assertEqual(conn.send(blob).status, 200) + + def test_heartbeat_keeps_it_alive(self): + conn = self._conn() + self.assertTrue(conn.heartbeat("/ping", "SID=x")) + + def test_send_before_connect_raises(self): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False) + blob = RequestBlob(host="127.0.0.1", path="/x", body="", headers={}) + with self.assertRaises(RuntimeError): + conn.send(blob) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +python3 -m unittest tests.test_transport -v +``` + +Expected: `ModuleNotFoundError: No module named 'fastform.transport'` + +- [ ] **Step 3: Write the implementation** + +Create `fastform/transport.py`: + +```python +"""Hold a connection open so sending costs a socket write, not a handshake. + +A cold DNS + TCP + TLS setup to docs.google.com on residential internet is +150-350ms and highly variable — bigger and jitterier than anything else in the +path. Connecting early and heartbeating turns that into zero. +""" + +import http.client +import time +from dataclasses import dataclass + +from fastform.compile import RequestBlob +from fastform.cookies import USER_AGENT + + +@dataclass +class Response: + status: int + location: str | None + body: bytes + elapsed_ms: float + + +class WarmConnection: + """One pre-established HTTP connection, reused across requests.""" + + def __init__( + self, + host: str, + port: int | None = None, + timeout: float = 15.0, + use_tls: bool = True, + ): + self.host = host + self.port = port + self.timeout = timeout + self.use_tls = use_tls + self._conn: http.client.HTTPConnection | None = None + + def connect(self) -> None: + cls = ( + http.client.HTTPSConnection if self.use_tls else http.client.HTTPConnection + ) + self._conn = cls(self.host, port=self.port, timeout=self.timeout) + self._conn.connect() + + def close(self) -> None: + if self._conn is not None: + self._conn.close() + self._conn = None + + def _request(self, method, path, body, headers) -> Response: + if self._conn is None: + raise RuntimeError("connect() must be called before sending") + started = time.perf_counter_ns() + self._conn.request(method, path, body=body, headers=headers) + resp = self._conn.getresponse() + payload = resp.read() # must drain fully or the connection cannot be reused + elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000 + return Response( + status=resp.status, + location=resp.getheader("Location"), + body=payload, + elapsed_ms=elapsed_ms, + ) + + def get(self, path: str, cookie: str) -> Response: + return self._request( + "GET", path, None, {"Cookie": cookie, "User-Agent": USER_AGENT} + ) + + def send(self, blob: RequestBlob) -> Response: + return self._request("POST", blob.path, blob.body, blob.headers) + + def heartbeat(self, path: str, cookie: str) -> bool: + """Keep the socket and any NAT mapping alive. Reconnects on failure.""" + try: + self.get(path, cookie) + return True + except (OSError, http.client.HTTPException): + self.close() + try: + self.connect() + return True + except OSError: + return False +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +python3 -m unittest tests.test_transport -v +``` + +Expected: `Ran 5 tests` … `OK` + +- [ ] **Step 5: Commit** + +```bash +git add fastform/transport.py tests/test_transport.py +git commit -m "Add warm connection transport" +``` + +--- + +### Task 11: The single-shot sender + +**Files:** +- Create: `fastform/fire.py` +- Create: `tests/test_fire.py` +- Modify: `README.md` + +**Interfaces:** +- Consumes: every module from Tasks 2, 5, 6, 7, 9, 10. +- Produces: `python3 -m fastform.fire --url ` — the deliverable for Phases 0–4. + +**This sender fires exactly once.** No loop, no resend path. It cannot double-submit because there is no code that sends twice. Retry discipline arrives in spec Phase 6, after calibration measures what is actually safe. Until then a timed-out send is a loss and the operator falls back to the browser. + +- [ ] **Step 1: Write the failing test** + +Only the pure decision logic is unit-tested; the network path is verified manually in Step 5. + +Create `tests/test_fire.py`: + +```python +import unittest + +from fastform.fire import ExitCode, decide_exit +from fastform.classify import Outcome + + +class TestDecideExit(unittest.TestCase): + def test_recorded_is_success(self): + self.assertEqual(decide_exit(Outcome.RECORDED), ExitCode.OK) + + def test_already_responded_is_success(self): + self.assertEqual(decide_exit(Outcome.ALREADY_RESPONDED), ExitCode.OK) + + def test_rejected_is_payload_error(self): + self.assertEqual(decide_exit(Outcome.REJECTED), ExitCode.REJECTED) + + def test_closed_is_its_own_code(self): + self.assertEqual(decide_exit(Outcome.CLOSED), ExitCode.CLOSED) + + def test_auth_fail_is_its_own_code(self): + self.assertEqual(decide_exit(Outcome.AUTH_FAIL), ExitCode.AUTH) + + def test_unknown_is_undetermined(self): + self.assertEqual(decide_exit(Outcome.UNKNOWN), ExitCode.UNDETERMINED) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +python3 -m unittest tests.test_fire -v +``` + +Expected: `ModuleNotFoundError: No module named 'fastform.fire'` + +- [ ] **Step 3: Write the implementation** + +Create `fastform/fire.py`: + +```python +"""Fetch a form, fill it, submit it once, and report what happened. + +Deliberately single-shot: there is no retry path anywhere in this file. At this +stage nothing has measured whether a second POST overwrites the first, so the +only safe number of sends is one. Spec Phase 6 replaces this with measured +retry discipline. +""" + +import argparse +import subprocess +import sys +import urllib.parse +from enum import IntEnum + +from fastform import compile as compile_mod +from fastform import classify as classify_mod +from fastform import cookies, match, parse +from fastform.transport import WarmConnection + + +class ExitCode(IntEnum): + OK = 0 + REJECTED = 1 + NOT_PARSEABLE = 2 + CLOSED = 3 + AUTH = 4 + UNDETERMINED = 5 + GAPS = 6 + + +_EXIT_BY_OUTCOME = { + classify_mod.Outcome.RECORDED: ExitCode.OK, + classify_mod.Outcome.ALREADY_RESPONDED: ExitCode.OK, + classify_mod.Outcome.REJECTED: ExitCode.REJECTED, + classify_mod.Outcome.CLOSED: ExitCode.CLOSED, + classify_mod.Outcome.AUTH_FAIL: ExitCode.AUTH, + classify_mod.Outcome.UNKNOWN: ExitCode.UNDETERMINED, +} + + +def decide_exit(outcome: "classify_mod.Outcome") -> ExitCode: + return _EXIT_BY_OUTCOME[outcome] + + +def open_in_browser(url: str) -> None: + """Start the browser fallback loading in parallel. Costs nothing if unused.""" + try: + subprocess.Popen( + ["open", url], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + except OSError: + pass + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(prog="fastform.fire") + ap.add_argument("--url", required=True) + ap.add_argument( + "--no-browser", action="store_true", help="skip opening the fallback tab" + ) + ap.add_argument( + "--force", action="store_true", help="submit even if required answers are missing" + ) + args = ap.parse_args(argv) + + if not args.no_browser: + open_in_browser(args.url) + + cookie = cookies.load() + parts = urllib.parse.urlsplit(args.url) + + conn = WarmConnection(parts.netloc) + conn.connect() + try: + path = parts.path + ("?" + parts.query if parts.query else "") + page = conn.get(path, cookie) + + try: + spec = parse.parse(page.body, args.url) + except parse.NotParseable as exc: + print(f"NOT PARSEABLE: {exc}") + print("Form is closed or deleted. Use the browser tab.") + return int(ExitCode.NOT_PARSEABLE) + + filled = match.fill(spec, match.load_bank()) + + if filled.blockers: + print(f"BLOCKED: {len(filled.blockers)} question(s) the direct POST cannot fill:") + for question in filled.blockers: + print(f" type={question.type} {question.text}") + print("Use the browser tab.") + return int(ExitCode.GAPS) + + if filled.gaps and not args.force: + print(f"GAPS: {len(filled.gaps)} required question(s) unmatched:") + for question in filled.gaps: + print(f" entry.{question.entry_id} {question.text}") + print("\nAdd them to config/answers.toml, or re-run with --force.") + return int(ExitCode.GAPS) + + blob = compile_mod.build(spec, filled, cookie) + result = conn.send(blob) + outcome = classify_mod.classify(result.status, result.location, result.body) + + print(f"outcome {outcome.value}") + print(f"status {result.status}") + print(f"elapsed {result.elapsed_ms:.1f} ms") + print(f"confidence {filled.confidence:.2f}") + + if outcome is classify_mod.Outcome.UNKNOWN: + print("\nSubmission status is UNDETERMINED. Check the response sheet.") + + return int(decide_exit(outcome)) + finally: + conn.close() + + +if __name__ == "__main__": + sys.exit(main()) +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +python3 -m unittest tests.test_fire -v +``` + +Expected: `Ran 6 tests` … `OK` + +- [ ] **Step 5: Verify end to end against F2** + +F2 has limit-to-1 off, so this is repeatable. + +```bash +cd /Users/supa/projects/active/fastform +python3 -m fastform.fire --url '' --no-browser +echo "exit=$?" +``` + +Expected: `outcome RECORDED`, an elapsed time in the low hundreds of milliseconds, and `exit=0`. + +Confirm a new row appeared in F2's Responses tab. **This is the real check** — the classifier saying `RECORDED` is not proof on its own. + +- [ ] **Step 6: Verify the gap path** + +```bash +python3 -m fastform.fire --url '' --no-browser +echo "exit=$?" +``` + +Expected: a `GAPS:` listing (F3 has questions your answer bank does not cover, such as `Rate your confidence`) and `exit=6`. Nothing was submitted. + +- [ ] **Step 7: Verify the blocker path** + +```bash +python3 -m fastform.fire --url '' --no-browser +echo "exit=$?" +``` + +Expected: a `BLOCKED:` line naming the file upload, and `exit=6`. Nothing was submitted. + +- [ ] **Step 8: Run the whole suite** + +```bash +python3 -m unittest discover -s tests -v +``` + +Expected: every test passes, `OK`. + +- [ ] **Step 9: Update the README** + +Replace `README.md` with: + +```markdown +# fastform + +Submits a Google Form fast, by POSTing directly to `formResponse` instead of +driving a browser. + +Design: `docs/superpowers/specs/2026-08-05-fastform-design.md` +Plan: `docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md` + +## Setup + +1. Sign into Google in Chrome. +2. Open any Google Form, DevTools → Network → copy the `Cookie:` request + header, and paste it into `config/cookies.txt` (untracked). +3. Put your answers in `config/answers.toml`. + +Verify the cookie works: + +```bash +python3 -c "from fastform import cookies; print(cookies.verify(cookies.load()))" +``` + +Expected: `(True, 'your@email')` + +## Use + +```bash +python3 tools/dryrun.py --url # build the request, send nothing +python3 -m fastform.fire --url # fill and submit, once +``` + +`fire` submits exactly once and never retries. Exit codes: `0` recorded, +`1` rejected, `2` unparseable, `3` closed, `4` auth failed, `5` undetermined, +`6` gaps or blockers (nothing sent). + +## Tests + +```bash +python3 -m unittest discover -s tests -v +``` + +## Status + +Spec Phases 0–4 complete. Calibration (Phase 5), retry discipline (Phase 6), +and the daemon with clipboard intake and the hold state (Phase 7) are not built. +``` + +- [ ] **Step 10: Commit** + +```bash +git add fastform/fire.py tests/test_fire.py README.md +git commit -m "Add single-shot sender CLI" +``` + +--- + +## Done criteria + +All of these must hold before this plan is considered complete: + +1. `python3 -m unittest discover -s tests -v` passes with no failures. +2. `python3 tools/curl_diff.py --curl fixtures/f1-manual-submit.curl.txt --url ` reports `MISSING — none`. +3. `python3 -m fastform.fire --url ` exits `0` and a new row appears in F2's Responses tab. +4. `python3 -m fastform.fire --url ` exits `6` and reports the file upload as a blocker, without submitting. +5. `git status` shows no untracked `config/cookies.txt` or `fixtures/*.curl.txt`. + +## What this plan does not build + +Carried forward to the next plan, after calibration supplies the numbers: + +- Experiments E3–E9 and the `calibrate.py` harness (spec Phase 5) +- `strategy.py`: retry, the timeout probe, stop conditions (spec Phase 6) +- `intake.py`, `tui.py`, `daemon.py`, `ARMED_HOLDING`, the confirm gate (spec Phase 7) +- Rehearsals (spec Phase 8) From 094356dba123b3f55e7b04e8dc170e0b4c4f1dcc Mon Sep 17 00:00:00 2001 From: ohm Date: Wed, 5 Aug 2026 21:33:29 +0700 Subject: [PATCH 05/24] Fix plan sequencing bugs that would have failed the E1 gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deep review of the plan found two defects that would have broken execution: 1. F1 was created with limit-to-one ON and then manually submitted to capture the reference cURL. Task 3's E1 gate POSTs to that same form from the same account — exactly what limit-to-one blocks. The gate would have returned "already responded", no row would have appeared, and the plan instructs the operator to conclude the design is unviable and stop. The same root cause crashed Task 8's curl_diff, whose GET would hit the already-responded page and raise NotParseable unhandled. 2. F1 disabled email collection and relied on limit-to-one for sign-in, while F2 was specified with limit-to-one off and "Responder input" — which does not force sign-in and contradicts F1. F2 would have required no auth, and every end-to-end check in Task 11 ran against it, so the cookie path went unexercised past Task 3. Both fixed by the same restructure: F1 forces sign-in via Collect email addresses -> Verified and keeps limit-to-one OFF, flipping it on only for Task 9's already-responded capture. That removes F2's reason to exist in this plan. Added an incognito check so a form that does not require sign-in is caught immediately. Also: - Cookie file must hold the bare value; the shell curl does not strip a "Cookie:" prefix the way the Python loader does, and the doubled header silently yields a signed-out page. - --drop-required now drops exactly one required field rather than emptying the payload, which risks being treated as a draft rather than a rejection. - Dropped the authfail fixture: AUTH_FAIL is detected from status and Location, neither of which a saved body preserves, and capturing it meant temporarily overwriting the real cookie file. - Task 10 now states plainly that Phase 4 gets no warm-pool benefit; the real gain is GET and POST sharing one connection. - fire.py distinguishes a redirect from a closed form, so a forms.gle link reports the redirect target instead of "form is closed". - Done criterion 5 checked git status for ignored files, which never shows them; now checks git ls-files. - Named BLOCKER_TYPES/CHOICE_TYPES in the Task 5/6 interface blocks. Re-extracted and re-ran all plan code: 40 tests pass, every module compiles, and the new drop-one-required path was exercised directly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- .../plans/2026-08-05-fastform-phases-0-4.md | 181 +++++++++++++----- 1 file changed, 132 insertions(+), 49 deletions(-) diff --git a/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md b/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md index 383b993..7cd671e 100644 --- a/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md +++ b/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md @@ -67,16 +67,33 @@ __pycache__/ This is manual work in a browser. Go to https://forms.google.com and create a form with these exact properties: - Title: `FastForm F1` -- Settings → Responses → **Collect email addresses: off** -- Settings → Responses → **Limit to 1 response: ON** (this forces sign-in) +- Settings → Responses → **Collect email addresses: Verified** +- Settings → Responses → **Limit to 1 response: OFF** - One section only. - Three questions, all **Short answer**, all **Required**: 1. `What is your full name?` 2. `What is your email address?` 3. `What is your phone number?` +**Why these two settings matter.** "Verified" is what forces sign-in, so the +authenticated POST path actually gets exercised — and unlike "Responder input" +it does not add a question we would have to answer, because Google fills the +address from the session. Limit-to-one stays **off** so F1 remains submittable +for the whole plan; Tasks 3, 8, 9, and 11 all submit to it repeatedly. Task 9 +turns limit-to-one on temporarily, at the one point where it is needed. + Click **Send** → link icon → copy the URL. Save it somewhere you can paste later; it looks like `https://docs.google.com/forms/d/e//viewform`. +- [ ] **Step 3a: Confirm the form really does require sign-in** + +Open the F1 URL in a **private/incognito window**. + +Expected: Google asks you to sign in before showing the form. If the form +renders without asking, "Verified" did not take effect — go back to +Settings → Responses and set it. Everything downstream assumes an authenticated +form, and a form that does not need auth would let the whole plan pass while +proving nothing. + - [ ] **Step 4: Submit F1 once by hand and capture the request** Open the F1 URL in Chrome, signed into the Google account you will compete with. @@ -98,7 +115,7 @@ echo "fixtures/*.curl.txt" >> .gitignore In the same DevTools Network panel, click the `formResponse` request → **Headers** → **Request Headers** → find `Cookie:`. Copy the entire value (it is long). -Write it to `config/cookies.txt` as a single line. The leading `Cookie:` is optional — Task 2 strips it either way. +Write it to `config/cookies.txt` as a single line, **value only — do not include the leading `Cookie:`**. The Python loader in Task 2 strips that prefix as a safety net, but the shell `curl` in the next step does not, and a doubled `Cookie: Cookie:` header silently produces a signed-out page. - [ ] **Step 6: Save the form page HTML as a fixture** @@ -445,6 +462,9 @@ Expected: `status=200` and a non-trivial byte count. A `status=302` with a `loca Open F1 in Google Forms → **Responses** tab. +F1 has limit-to-one **off**, so this is a second row alongside the manual +submission from Task 1 — that is expected, and it is why limit-to-one is off. + **PASS:** a new row appeared containing `E1 Gate Test`. The design is viable. Continue. **FAIL:** no new row. Stop here. Capture what you can and re-plan: @@ -464,7 +484,7 @@ git commit -m "Add E1 gate spike: cookie-only POST records a response" --- -### Task 4: Create test forms F2, F3, F4 +### Task 4: Create test forms F3 and F4 **Files:** - Modify: `fixtures/README.md` @@ -477,15 +497,18 @@ git commit -m "Add E1 gate spike: cookie-only POST records a response" Deferred until now on purpose: if Task 3 had failed, these forms would have been wasted effort. -- [ ] **Step 1: Create F2** - -Duplicate F1 (Forms → ⋮ → Make a copy), title it `FastForm F2`, and change one setting: **Limit to 1 response: OFF**. Keep sign-in required by leaving "Collect email addresses" set to **Responder input**. +The spec's test bed pairs F1 (limit-to-one ON) with F2 (limit-to-one OFF). +This plan inverts that: F1 runs with limit-to-one **off** so it stays +submittable throughout, and Task 9 flips it on for one capture. That makes a +separate always-off F2 redundant here. The permanently-ON variant the spec calls +F1 is what Phase 5 will need, for E3's comparison of overwrite behavior with the +setting on versus off — it gets created in that plan. -Save its viewform URL. +**Both forms below use Settings → Responses → Collect email addresses: Verified**, for the same reason F1 does — it forces sign-in without adding a question. -- [ ] **Step 2: Create F3 — multi-section, mixed types** +- [ ] **Step 1: Create F3 — multi-section, mixed types** -Title `FastForm F3`. Sign-in required, limit-to-1 off. Three sections: +Title `FastForm F3`. Collect email addresses: **Verified**. Limit to 1 response: **OFF**. Three sections: **Section 1:** - `What is your full name?` — Short answer, Required @@ -503,9 +526,9 @@ Title `FastForm F3`. Sign-in required, limit-to-1 off. Three sections: Save its viewform URL. -- [ ] **Step 3: Create F4 — validation and a file upload** +- [ ] **Step 2: Create F4 — validation and a file upload** -Title `FastForm F4`. Sign-in required. One section: +Title `FastForm F4`. Collect email addresses: **Verified**. One section: - `Numeric code` — Short answer, Required, Response validation → Number → Between → `1000` and `9999` - `Contact email` — Short answer, Required, Response validation → Text → Email address @@ -513,7 +536,7 @@ Title `FastForm F4`. Sign-in required. One section: Save its viewform URL. -- [ ] **Step 4: Capture F3 and F4 page fixtures** +- [ ] **Step 3: Capture F3 and F4 page fixtures** ```bash cd /Users/supa/projects/active/fastform @@ -524,7 +547,7 @@ grep -c FB_PUBLIC_LOAD_DATA_ fixtures/f3-viewform.html fixtures/f4-viewform.html Expected: `fixtures/f3-viewform.html:1` and `fixtures/f4-viewform.html:1` -- [ ] **Step 5: Record the URLs** +- [ ] **Step 4: Record the URLs** Append to `fixtures/README.md`: @@ -532,10 +555,11 @@ Append to `fixtures/README.md`: ## Test forms +All require sign-in via Collect email addresses → Verified. + | Form | Purpose | URL | |------|---------|-----| -| F1 | limit-to-1 ON, 3 required short answers | | -| F2 | limit-to-1 OFF, otherwise identical to F1 | | +| F1 | 3 required short answers; limit-to-1 OFF (temporarily ON in Task 9) | | | F3 | 3 sections, mixed question types | | | F4 | validation rules + file upload | | @@ -543,11 +567,11 @@ Append to `fixtures/README.md`: - `f4-viewform.html` — validation and file-upload fixture for blocker detection. ``` -- [ ] **Step 6: Commit** +- [ ] **Step 5: Commit** ```bash git add fixtures/README.md fixtures/f1-viewform.html fixtures/f3-viewform.html fixtures/f4-viewform.html -git commit -m "Add test forms F2-F4 and page fixtures" +git commit -m "Add test forms F3-F4 and page fixtures" ``` --- @@ -565,7 +589,9 @@ git commit -m "Add test forms F2-F4 and page fixtures" - `FormSpec` dataclass: `form_id: str`, `action_path: str`, `page_count: int`, `questions: list[Question]`; properties `page_history: str`, `blockers: list[Question]` - `parse(html: bytes, url: str) -> FormSpec` - `form_id_from_url(url: str) -> str` - - `TYPE_FILE_UPLOAD`, `TYPE_SECTION`, and the other type constants. + - `NotParseable` exception + - `BLOCKER_TYPES: frozenset[int]` and `CHOICE_TYPES: frozenset[int]` — Task 6 imports both by name + - `TYPE_SHORT_ANSWER`, `TYPE_PARAGRAPH`, `TYPE_MULTIPLE_CHOICE`, `TYPE_DROPDOWN`, `TYPE_CHECKBOXES`, `TYPE_LINEAR_SCALE`, `TYPE_GRID`, `TYPE_SECTION`, `TYPE_DATE`, `TYPE_TIME`, `TYPE_FILE_UPLOAD` **Important:** the index positions inside `FB_PUBLIC_LOAD_DATA_` below are my best understanding, not verified fact. Step 1 makes you check them against your own fixture before writing code. If they differ, adjust the constants — the tests assert on question text and entry IDs you control, so they will tell you when you have it right. @@ -860,7 +886,7 @@ git commit -m "Add form structure parser" - Create: `tests/test_match.py` **Interfaces:** -- Consumes: `Question`, `FormSpec`, `CHOICE_TYPES` from `fastform.parse`. +- Consumes: `Question`, `FormSpec`, `BLOCKER_TYPES`, `CHOICE_TYPES` from `fastform.parse`. - Produces: - `FilledForm` dataclass: `pairs: list[tuple[str, str]]`, `gaps: list[Question]`, `blockers: list[Question]`, `confidence: float` - `load_bank(path: Path | None = None) -> list[dict]` @@ -1575,9 +1601,9 @@ def main() -> int: ap.add_argument("--url", required=True) ap.add_argument("--name", required=True, help="fixture suffix, e.g. success") ap.add_argument( - "--drop-required", + "--drop-one-required", action="store_true", - help="omit every required field, to provoke a validation rejection", + help="omit a single required field, to provoke a validation rejection", ) args = ap.parse_args() @@ -1592,8 +1618,19 @@ def main() -> int: return 0 filled = match.fill(spec, match.load_bank()) - if args.drop_required: - filled.pairs = [] + + if args.drop_one_required: + # Drop exactly one required answer rather than emptying the payload. + # An entirely empty POST risks being treated as a draft rather than a + # rejection, which would make the fixture the wrong shape. + required_ids = {q.entry_id for q in spec.questions if q.required} + for index, (entry_id, _) in enumerate(filled.pairs): + if entry_id in required_ids: + dropped = filled.pairs.pop(index) + print(f"dropped required field entry.{dropped[0]}") + break + else: + raise SystemExit("no required field was filled — nothing to drop") blob = compile_mod.build(spec, filled, cookie) conn = http.client.HTTPSConnection(blob.host, timeout=15) @@ -1617,37 +1654,53 @@ if __name__ == "__main__": - [ ] **Step 2: Capture all five fixtures** +All four go against F1, in this order. Two of them need a Forms setting toggled +first — do them in sequence, not out of order. + ```bash cd /Users/supa/projects/active/fastform -# 1. success — F2 has limit-to-1 off, so this is repeatable -python3 tools/capture_responses.py --url '' --name success +# 1. success — F1 has limit-to-1 off, so this is repeatable +python3 tools/capture_responses.py --url '' --name success + +# 2. rejected — one required field omitted +python3 tools/capture_responses.py --url '' --name rejected --drop-one-required +``` -# 2. rejected — same form, required fields stripped -python3 tools/capture_responses.py --url '' --name rejected --drop-required +**3. closed.** In the Forms UI, turn **OFF** F1's "Accepting responses". Then: -# 3. already-responded — F1 has limit-to-1 on and already has your E1 row -python3 tools/capture_responses.py --url '' --name already +```bash +python3 tools/capture_responses.py --url '' --name closed +``` -# 4. closed — turn OFF "Accepting responses" on F2 in the Forms UI first, then: -python3 tools/capture_responses.py --url '' --name closed -# then turn accepting responses back ON +The tool takes its not-parseable branch here and saves the GET body — that page +*is* the closed-form page, which is what the classifier needs. Now turn +**Accepting responses back ON.** -# 5. auth-fail -printf 'SID=definitely-not-valid' > /tmp/bad-cookies.txt -cp config/cookies.txt /tmp/real-cookies.txt -cp /tmp/bad-cookies.txt config/cookies.txt -python3 tools/capture_responses.py --url '' --name authfail -cp /tmp/real-cookies.txt config/cookies.txt +**4. already-responded.** In the Forms UI, turn **ON** F1's "Limit to 1 response". +The rows already in the sheet make your account count as having responded. Then: + +```bash +python3 tools/capture_responses.py --url '' --name already ``` -Verify all five exist: +This also takes the not-parseable branch, because an already-responded form does +not render its questions. Now turn **Limit to 1 response back OFF**, or Task 11's +end-to-end check will fail. + +There is deliberately no `authfail` fixture. `AUTH_FAIL` is detected from the +redirect status and `Location` header, neither of which a saved body preserves — +the fixture would be unusable, and the structural unit test in Step 4 already +covers it. Capturing it would also mean temporarily overwriting your real +`config/cookies.txt`, which is a needless risk for an artifact nothing reads. + +Verify all four exist: ```bash ls -la fixtures/resp-*.bin ``` -Expected: `resp-success.bin`, `resp-rejected.bin`, `resp-already.bin`, `resp-closed.bin`, `resp-authfail.bin` +Expected: `resp-success.bin`, `resp-rejected.bin`, `resp-closed.bin`, `resp-already.bin` (plus `resp-e1.bin` from Task 3). - [ ] **Step 3: Inspect what actually distinguishes them** @@ -1673,7 +1726,9 @@ from fastform.classify import Outcome, classify FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" -# Each fixture was captured from a real submission in Task 9 Step 2. +# Captured in Task 9 Step 2. The first two are POST responses; the last two are +# GET bodies, because a closed or already-responded form never renders far +# enough to POST. classify() handles both, which is why this list is uniform. CASES = [ ("resp-success.bin", 200, None, Outcome.RECORDED), ("resp-rejected.bin", 200, None, Outcome.REJECTED), @@ -1812,6 +1867,15 @@ git commit -m "Add response classifier with captured fixtures" - `Response` dataclass: `status: int`, `location: str | None`, `body: bytes`, `elapsed_ms: float` - `WarmConnection` class: `connect()`, `heartbeat()`, `get(path, cookie) -> Response`, `send(blob) -> Response`, `close()` +**Be clear about what this buys at Phase 4.** A hand-run tool connects on demand, +so the connection is cold on first use and none of the spec's warm-pool latency +win applies yet — that needs a long-running daemon, which is Phase 7. What this +does buy immediately is real: `fire.py` performs its GET and its POST on the +same connection, so the POST costs no handshake at all. One saved TLS setup on +the request that matters. The pool of 2–3 connections and the heartbeat loop are +Phase 7's job; `heartbeat()` exists here because it is small and belongs with +the class it operates on, not because anything in this plan calls it in a loop. + - [ ] **Step 1: Write the failing test** Uses a local `http.server`, so no external network. @@ -2163,7 +2227,14 @@ def main(argv=None) -> int: spec = parse.parse(page.body, args.url) except parse.NotParseable as exc: print(f"NOT PARSEABLE: {exc}") - print("Form is closed or deleted. Use the browser tab.") + if page.status in (301, 302, 303, 307, 308): + # A shortened forms.gle link lands here. Redirects are not + # followed at this phase — say so rather than blaming the form. + print(f"The URL redirected to: {page.location}") + print("Pass the full https://docs.google.com/forms/d/e/.../viewform URL.") + else: + print("Form is closed, already responded to, or deleted.") + print("Use the browser tab.") return int(ExitCode.NOT_PARSEABLE) filled = match.fill(spec, match.load_bank()) @@ -2211,19 +2282,19 @@ python3 -m unittest tests.test_fire -v Expected: `Ran 6 tests` … `OK` -- [ ] **Step 5: Verify end to end against F2** +- [ ] **Step 5: Verify end to end against F1** -F2 has limit-to-1 off, so this is repeatable. +F1 has limit-to-1 off, so this is repeatable. If Task 9 left it on, turn it off first. ```bash cd /Users/supa/projects/active/fastform -python3 -m fastform.fire --url '' --no-browser +python3 -m fastform.fire --url '' --no-browser echo "exit=$?" ``` Expected: `outcome RECORDED`, an elapsed time in the low hundreds of milliseconds, and `exit=0`. -Confirm a new row appeared in F2's Responses tab. **This is the real check** — the classifier saying `RECORDED` is not proof on its own. +Confirm a new row appeared in F1's Responses tab. **This is the real check** — the classifier saying `RECORDED` is not proof on its own. - [ ] **Step 6: Verify the gap path** @@ -2317,15 +2388,27 @@ All of these must hold before this plan is considered complete: 1. `python3 -m unittest discover -s tests -v` passes with no failures. 2. `python3 tools/curl_diff.py --curl fixtures/f1-manual-submit.curl.txt --url ` reports `MISSING — none`. -3. `python3 -m fastform.fire --url ` exits `0` and a new row appears in F2's Responses tab. +3. `python3 -m fastform.fire --url ` exits `0` and a new row appears in F1's Responses tab. 4. `python3 -m fastform.fire --url ` exits `6` and reports the file upload as a blocker, without submitting. -5. `git status` shows no untracked `config/cookies.txt` or `fixtures/*.curl.txt`. +5. No secrets are tracked. Verify with `git ls-files | grep -E 'cookies\.txt|\.curl\.txt'` — expected: no output. (`git status` would not reveal this; ignored files do not appear as untracked.) +6. F1 is left with **Limit to 1 response OFF** and **Accepting responses ON**, so it stays usable. ## What this plan does not build Carried forward to the next plan, after calibration supplies the numbers: +- A second test form with limit-to-one permanently ON, needed by E3's overwrite comparison and not by this plan - Experiments E3–E9 and the `calibrate.py` harness (spec Phase 5) - `strategy.py`: retry, the timeout probe, stop conditions (spec Phase 6) - `intake.py`, `tui.py`, `daemon.py`, `ARMED_HOLDING`, the confirm gate (spec Phase 7) - Rehearsals (spec Phase 8) + +Two things from the spec's ARMING section are also absent here, and `fire.py` +reports them rather than handling them: + +- **URL normalization and short-link resolution.** `forms.gle` links, chat-paste + punctuation, and redirect wrappers are not handled. Pass the full + `docs.google.com/forms/d/e/.../viewform` URL. This belongs with `intake.py` in + Phase 7; `fire.py` prints the redirect target so the failure is legible. +- **The multi-connection warm pool and heartbeat loop.** See the note in Task 10 + — a hand-run tool has no idle period to warm through. From d2dc70ddc320fd239e728a209d7e0b76c356a49a Mon Sep 17 00:00:00 2001 From: ohm Date: Wed, 5 Aug 2026 22:22:24 +0700 Subject: [PATCH 06/24] Scaffold package layout and answer bank --- .gitignore | 5 +++++ config/answers.toml | 17 +++++++++++++++++ fastform/__init__.py | 0 fixtures/README.md | 10 ++++++++++ tests/__init__.py | 0 5 files changed, 32 insertions(+) create mode 100644 .gitignore create mode 100644 config/answers.toml create mode 100644 fastform/__init__.py create mode 100644 fixtures/README.md create mode 100644 tests/__init__.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4f4b14d --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +config/cookies.txt +__pycache__/ +*.pyc +.DS_Store +fixtures/*.curl.txt diff --git a/config/answers.toml b/config/answers.toml new file mode 100644 index 0000000..488f332 --- /dev/null +++ b/config/answers.toml @@ -0,0 +1,17 @@ +# Answer bank. Each entry matches a question by text and supplies a value. +# match — list of phrases; scored against question text (see fastform/match.py) +# regex — optional; a hit here is an exact match, outranking everything +# value — what gets submitted + +[[answer]] +match = ["full name", "your name", "name"] +value = "Test Person" + +[[answer]] +match = ["email address", "email"] +regex = "e-?mail" +value = "test@example.com" + +[[answer]] +match = ["phone number", "phone", "mobile"] +value = "0800000000" diff --git a/fastform/__init__.py b/fastform/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fixtures/README.md b/fixtures/README.md new file mode 100644 index 0000000..b743de5 --- /dev/null +++ b/fixtures/README.md @@ -0,0 +1,10 @@ +# Fixtures + +Captured artifacts used as test data. Nothing here is generated by code. + +- `f1-viewform.html` — F1 form page, signed in. Input for parse tests. +- `f1-manual-submit.curl.txt` — DevTools "copy as cURL" of a real manual + submission to F1. Ground truth for the Task 8 request diff. Untracked: + contains cookies. + +Response fixtures (`resp-*.bin`) are captured in Task 9. diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 From a8ab496f3563032b65b51000ac471259a8d2a469 Mon Sep 17 00:00:00 2001 From: ohm Date: Wed, 5 Aug 2026 22:25:51 +0700 Subject: [PATCH 07/24] Add cookie loading and freshness check Implement fastform.cookies module with load() and verify() functions. - load() reads and parses the Cookie header from config/cookies.txt - verify() checks that the cookie still authenticates against Google - USER_AGENT constant available for downstream tasks Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- fastform/cookies.py | 58 +++++++++++++++++++++++++++++++++++++++++++ tests/test_cookies.py | 38 ++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 fastform/cookies.py create mode 100644 tests/test_cookies.py diff --git a/fastform/cookies.py b/fastform/cookies.py new file mode 100644 index 0000000..2b53016 --- /dev/null +++ b/fastform/cookies.py @@ -0,0 +1,58 @@ +"""Load a Cookie header captured from Chrome DevTools and check it still authenticates. + +Cookies are captured by hand rather than decrypted out of Chrome's SQLite +store: decryption needs AES (a dependency), and Google session cookies last +months. See the plan's Scope Note. +""" + +import http.client +import re +from pathlib import Path + +USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36" +) + +DEFAULT_PATH = Path(__file__).resolve().parent.parent / "config" / "cookies.txt" + +_EMAIL_RE = re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+") + + +def load(path: Path | None = None) -> str: + """Return the bare Cookie header value, with any 'Cookie:' prefix removed.""" + path = path or DEFAULT_PATH + raw = path.read_text(encoding="utf-8").strip() + if raw.lower().startswith("cookie:"): + raw = raw.split(":", 1)[1].strip() + if not raw: + raise ValueError(f"{path} is empty — paste a Cookie header into it") + return raw + + +def verify(cookie_header: str) -> tuple[bool, str | None]: + """GET a signed-in-only page. Returns (signed_in, email_if_found). + + Signed-out sessions get redirected to accounts.google.com, which is a + structural signal and does not depend on the account's UI language. + """ + conn = http.client.HTTPSConnection("docs.google.com", timeout=15) + try: + conn.request( + "GET", + "/forms/u/0/", + headers={"Cookie": cookie_header, "User-Agent": USER_AGENT}, + ) + resp = conn.getresponse() + location = resp.getheader("Location") or "" + body = resp.read() + finally: + conn.close() + + if resp.status in (301, 302, 303, 307, 308) and "accounts.google.com" in location: + return False, None + if resp.status != 200: + return False, None + + match = _EMAIL_RE.search(body.decode("utf-8", "replace")) + return True, match.group(0) if match else None diff --git a/tests/test_cookies.py b/tests/test_cookies.py new file mode 100644 index 0000000..c7b86fb --- /dev/null +++ b/tests/test_cookies.py @@ -0,0 +1,38 @@ +import tempfile +import unittest +from pathlib import Path + +from fastform import cookies + + +class TestLoad(unittest.TestCase): + def _write(self, text): + tmp = Path(tempfile.mkdtemp()) / "cookies.txt" + tmp.write_text(text, encoding="utf-8") + return tmp + + def test_returns_bare_value(self): + path = self._write("SID=abc; HSID=def\n") + self.assertEqual(cookies.load(path), "SID=abc; HSID=def") + + def test_strips_cookie_prefix(self): + path = self._write("Cookie: SID=abc; HSID=def") + self.assertEqual(cookies.load(path), "SID=abc; HSID=def") + + def test_strips_lowercase_prefix(self): + path = self._write("cookie: SID=abc") + self.assertEqual(cookies.load(path), "SID=abc") + + def test_empty_file_raises(self): + path = self._write(" \n") + with self.assertRaises(ValueError): + cookies.load(path) + + def test_missing_file_raises(self): + path = Path(tempfile.mkdtemp()) / "nope.txt" + with self.assertRaises(FileNotFoundError): + cookies.load(path) + + +if __name__ == "__main__": + unittest.main() From 967cce1357b931fd1fc58c836b2d7b94cfbe4034 Mon Sep 17 00:00:00 2001 From: ohm Date: Wed, 5 Aug 2026 22:30:06 +0700 Subject: [PATCH 08/24] Correct overclaimed cookie-challenge mitigation in spec Task 2's review established that verify() cannot distinguish a captcha or consent interstitial served with HTTP 200 from a signed-in page, so the Known Risks entry claiming the boot-time authenticated GET mitigates the 'verify it's you' challenge was overstating what the check can deliver. Closing the gap needs a captured sample of such a page to derive a marker from; guessing at markers without one would be worse than the stated gap. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- docs/superpowers/specs/2026-08-05-fastform-design.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/specs/2026-08-05-fastform-design.md b/docs/superpowers/specs/2026-08-05-fastform-design.md index 34f810c..1fa9680 100644 --- a/docs/superpowers/specs/2026-08-05-fastform-design.md +++ b/docs/superpowers/specs/2026-08-05-fastform-design.md @@ -509,8 +509,15 @@ time runs short: the LAN listener (stdin works), the ctypes clipboard binding - **A file upload question breaks the direct-POST path entirely.** Detected at parse time as a blocker and routed to the browser. Detection is cheap; solving it is not, and would lose the race anyway. -- **Google cookies may hit a "verify it's you" challenge** on the day. Mitigated - by the boot-time authenticated GET, run well before the event. +- **Google cookies may hit a "verify it's you" challenge** on the day. The + boot-time authenticated GET only partly mitigates this. It detects a signed-out + session by the redirect to `accounts.google.com`, but a captcha, consent, or + "unusual traffic" interstitial served with HTTP 200 is indistinguishable from a + signed-in page by status alone, and would be reported as authenticated. Closing + that gap needs a captured sample of such a page to derive a marker from, which + calibration should collect if one ever appears. Until then, treat a green + boot-time check as necessary but not sufficient, and confirm by eye that a form + actually renders. - **The form may be edited between an early leak and the announced open**, staling the entry IDs a held payload was built against. Mitigated by hold-time re-validation, which re-runs matching on a detected structural change. A From 1a8efda17436c96b953c9778dc130a969046abc4 Mon Sep 17 00:00:00 2001 From: ohm Date: Wed, 5 Aug 2026 22:31:46 +0700 Subject: [PATCH 09/24] Add unit tests for cookies.verify() verify() decides whether a stored cookie is still signed in, and every later task acts on that answer -- but it had zero test coverage. Add unit tests covering signed-in with/without an email in the body, redirect to accounts.google.com, redirect elsewhere (documents the actual fall-through behavior, not a redesigned one), and a bare 5xx status. All tests patch http.client.HTTPSConnection so no real network request occurs. Also note in verify()'s docstring that any HTTP 200 is currently read as signed in, so a captcha/interstitial served with status 200 would be misread as authenticated -- documentation only, no detection added. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- fastform/cookies.py | 5 +++ tests/test_cookies.py | 81 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/fastform/cookies.py b/fastform/cookies.py index 2b53016..03b5317 100644 --- a/fastform/cookies.py +++ b/fastform/cookies.py @@ -35,6 +35,11 @@ def verify(cookie_header: str) -> tuple[bool, str | None]: Signed-out sessions get redirected to accounts.google.com, which is a structural signal and does not depend on the account's UI language. + + Limitation: any HTTP 200 response is treated as signed in. A captcha or + "unusual traffic" interstitial served with status 200 would be + misread as an authenticated session -- this function does not attempt + to detect that case. """ conn = http.client.HTTPSConnection("docs.google.com", timeout=15) try: diff --git a/tests/test_cookies.py b/tests/test_cookies.py index c7b86fb..0a6b07a 100644 --- a/tests/test_cookies.py +++ b/tests/test_cookies.py @@ -1,6 +1,7 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import MagicMock, patch from fastform import cookies @@ -34,5 +35,85 @@ def test_missing_file_raises(self): cookies.load(path) +def _fake_connection(status, location=None, body=b""): + """Build a stand-in for http.client.HTTPSConnection's instance. + + Mimics the subset of the interface verify() touches: request(), + getresponse() (returning an object with .status, .getheader(), and + .read()), and close(). + """ + response = MagicMock() + response.status = status + response.getheader.side_effect = ( + lambda name, default=None: location if name == "Location" else default + ) + response.read.return_value = body + + conn = MagicMock() + conn.getresponse.return_value = response + return conn + + +class TestVerify(unittest.TestCase): + def _patch_connection(self, status, location=None, body=b""): + """Patch http.client.HTTPSConnection as seen from fastform.cookies + so verify() can never open a real socket, and return the fake + connection instance it will receive. + """ + conn = _fake_connection(status, location, body) + patcher = patch( + "fastform.cookies.http.client.HTTPSConnection", return_value=conn + ) + self.addCleanup(patcher.stop) + patcher.start() + return conn + + def test_200_with_email_is_signed_in(self): + self._patch_connection(200, body=b"Signed in as jane.doe@example.com today") + signed_in, email = cookies.verify("SID=abc") + self.assertTrue(signed_in) + self.assertEqual(email, "jane.doe@example.com") + + def test_200_without_email_is_signed_in_with_none(self): + self._patch_connection(200, body=b"no email on this page") + signed_in, email = cookies.verify("SID=abc") + self.assertTrue(signed_in) + self.assertIsNone(email) + + def test_302_to_accounts_google_is_signed_out(self): + self._patch_connection( + 302, location="https://accounts.google.com/ServiceLogin", body=b"" + ) + signed_in, email = cookies.verify("SID=abc") + self.assertFalse(signed_in) + self.assertIsNone(email) + + def test_302_to_other_location_is_also_signed_out(self): + # verify() only special-cases redirects whose Location contains + # accounts.google.com. A redirect anywhere else does not match that + # branch, but it still isn't status 200, so it falls through to the + # generic "status != 200" check and is reported as signed out -- + # not because the redirect target was actually inspected. + self._patch_connection( + 302, + location="https://docs.google.com/forms/u/0/somewhere-else", + body=b"", + ) + signed_in, email = cookies.verify("SID=abc") + self.assertFalse(signed_in) + self.assertIsNone(email) + + def test_non_200_non_redirect_status_is_signed_out(self): + self._patch_connection(500, body=b"Internal Server Error") + signed_in, email = cookies.verify("SID=abc") + self.assertFalse(signed_in) + self.assertIsNone(email) + + def test_connection_is_closed(self): + conn = self._patch_connection(200, body=b"no email here") + cookies.verify("SID=abc") + conn.close.assert_called_once() + + if __name__ == "__main__": unittest.main() From d8fb5220cb01a9547e739ba6a98c51e7af28464c Mon Sep 17 00:00:00 2001 From: ohm Date: Wed, 5 Aug 2026 23:02:13 +0700 Subject: [PATCH 10/24] Correct spec and plan from E1 gate evidence The E1 gate passed - a cookie-only POST does record a response - but running it disproved four assumptions that were load-bearing in the design. 1. COOKIES EXPIRE IN MINUTES, NOT MONTHS. A header copied from DevTools authenticated for ~7 minutes and was dead across all of Google within 20. Chrome rotates __Secure-1PSIDTS continuously (4 minutes old in the live profile, vs 35 hours for __Secure-1PSID); once it rotates the captured value stops working. The browser session stays healthy. "Extract cookies once, offline" cannot work, and neither can a boot-time check hours early nor ARMED_HOLDING on a static header. Reading Chrome's cookie store live is now the design, verified end to end with zero Python dependencies via Keychain plus an openssl subprocess. 2. THE POST NEEDS token AND tag. Both are per-page-load hidden inputs; dropping either returns 400. They are NOT reCAPTCHA - the page has no recaptcha, grecaptcha or botguard reference - so scraping them from the arming GET works. Their values are HTML-escaped in the page and must be unescaped, or the POST 400s even with correct field names. emailAddress is also required when the form collects the responder's address. 3. GOOGLE FORMS DOES NOT REDIRECT FOR SIGN-IN. A signed-out sign-in-required form returns 200 with the question structure fully intact, so parsing succeeds while unauthenticated. Detection must key on the data-sign-in-to-continue attribute, which is locale-independent. 4. FB_PUBLIC_LOAD_DATA_ IS PRESENT ON THE SUCCESS PAGE TOO. The heuristic "form re-rendered means rejected" reported REJECTED on a submission that had actually been recorded - the worst error available, since it invites a retry. The real discriminator is whether the response still contains a
element. The freebirdFormviewerViewResponse* classes the classifier keyed on do not exist in current markup. Adds real captured fixtures for the success and rejection responses. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- .../plans/2026-08-05-fastform-phases-0-4.md | 239 +++++++++++++++++- .../specs/2026-08-05-fastform-design.md | 63 ++++- fixtures/f1-viewform.html | 1 + fixtures/resp-rejected.bin | 1 + fixtures/resp-success.bin | 1 + 5 files changed, 285 insertions(+), 20 deletions(-) create mode 100644 fixtures/f1-viewform.html create mode 100644 fixtures/resp-rejected.bin create mode 100644 fixtures/resp-success.bin diff --git a/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md b/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md index 7cd671e..c463ed5 100644 --- a/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md +++ b/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md @@ -586,14 +586,23 @@ git commit -m "Add test forms F3-F4 and page fixtures" - Consumes: fixtures from Tasks 1 and 4. - Produces: - `Question` dataclass: `entry_id: str`, `text: str`, `type: int`, `required: bool`, `options: list[str]` - - `FormSpec` dataclass: `form_id: str`, `action_path: str`, `page_count: int`, `questions: list[Question]`; properties `page_history: str`, `blockers: list[Question]` + - `FormSpec` dataclass: `form_id: str`, `action_path: str`, `page_count: int`, `questions: list[Question]`, `hidden: dict[str, str]`; properties `page_history: str`, `blockers: list[Question]` + - `requires_sign_in(html: bytes) -> bool` + - `signed_in_email(html: bytes) -> str | None` + - `hidden_inputs(html: bytes) -> dict[str, str]` - `parse(html: bytes, url: str) -> FormSpec` - `form_id_from_url(url: str) -> str` - `NotParseable` exception - `BLOCKER_TYPES: frozenset[int]` and `CHOICE_TYPES: frozenset[int]` — Task 6 imports both by name - `TYPE_SHORT_ANSWER`, `TYPE_PARAGRAPH`, `TYPE_MULTIPLE_CHOICE`, `TYPE_DROPDOWN`, `TYPE_CHECKBOXES`, `TYPE_LINEAR_SCALE`, `TYPE_GRID`, `TYPE_SECTION`, `TYPE_DATE`, `TYPE_TIME`, `TYPE_FILE_UPLOAD` -**Important:** the index positions inside `FB_PUBLIC_LOAD_DATA_` below are my best understanding, not verified fact. Step 1 makes you check them against your own fixture before writing code. If they differ, adjust the constants — the tests assert on question text and entry IDs you control, so they will tell you when you have it right. +**The index positions inside `FB_PUBLIC_LOAD_DATA_` are verified** against the real F1 fixture: `item[1]` = title, `item[3]` = type, `item[4][0][0]` = entry id, `item[4][0][1]` = options, `item[4][0][2]` = required. Step 1 re-confirms them against your own capture. + +**Three facts established empirically during the E1 gate, which this task must encode:** + +1. **A signed-out page still contains the full question structure.** Google Forms does not redirect for a sign-in-required form — it returns HTTP 200 with `FB_PUBLIC_LOAD_DATA_` intact and the inputs disabled. So parsing succeeding proves nothing about being authenticated, and `requires_sign_in()` exists to catch that before a wasted POST. +2. **The signals are HTML attributes, not text.** `data-sign-in-to-continue="true"` appears only when signed out; `data-user-email-address="…"` carries the account identity when signed in and is empty otherwise. Attributes are locale-independent, which matters — the account under test renders in Thai. +3. **The POST needs `token` and `tag`, which live in hidden inputs** and change on every page load. Their values are **HTML-escaped in the page** and must be unescaped, or the POST returns 400 despite having the right field names. - [ ] **Step 1: Inspect the real structure first** @@ -678,6 +687,48 @@ class TestParseF1(unittest.TestCase): def test_no_blockers(self): self.assertEqual(self.spec.blockers, []) + def test_hidden_inputs_carry_token_and_tag(self): + self.assertIn("token", self.spec.hidden) + self.assertIn("tag", self.spec.hidden) + self.assertTrue(self.spec.hidden["token"]) + + def test_hidden_values_are_html_unescaped(self): + # partialResponse holds JSON, so it contains quotes that the page + # escapes as ". Leaving them escaped 400s the POST. + self.assertNotIn(""", self.spec.hidden.get("partialResponse", "")) + + +class TestSignInDetection(unittest.TestCase): + """Google Forms returns 200 for a signed-out sign-in-required form, with the + question structure fully present. Only an attribute distinguishes the two.""" + + SIGNED_OUT = b'
' + SIGNED_IN = b'
' + + def test_detects_signed_out(self): + self.assertTrue(parse.requires_sign_in(self.SIGNED_OUT)) + + def test_detects_signed_in(self): + self.assertFalse(parse.requires_sign_in(self.SIGNED_IN)) + + def test_extracts_email_when_signed_in(self): + self.assertEqual( + parse.signed_in_email(self.SIGNED_IN), "someone@example.com" + ) + + def test_no_email_when_signed_out(self): + self.assertIsNone(parse.signed_in_email(self.SIGNED_OUT)) + + +class TestHiddenInputs(unittest.TestCase): + def test_unescapes_values(self): + html = b'' + self.assertEqual(parse.hidden_inputs(html), {"x": 'a"b&c'}) + + def test_ignores_non_hidden_inputs(self): + html = b'' + self.assertEqual(parse.hidden_inputs(html), {}) + class TestParseF3(unittest.TestCase): @classmethod @@ -740,6 +791,7 @@ A closed form does not include that blob, which is why NotParseable exists and why the caller has to treat it as a distinct state rather than an error. """ +import html as html_mod import json import re from dataclasses import dataclass, field @@ -766,6 +818,12 @@ _LOAD_DATA_RE = re.compile( rb"FB_PUBLIC_LOAD_DATA_\s*=\s*(\[.*?\]);\s*", re.S ) _FORM_ID_RE = re.compile(r"/forms/d/e/([^/]+)/") +_HIDDEN_RE = re.compile( + rb']+type="hidden"[^>]+name="([^"]+)"[^>]+value="([^"]*)"' +) +#: Present only when the viewer is signed out of a sign-in-required form. +_SIGN_IN_RE = re.compile(rb'data-sign-in-to-continue="true"') +_VIEWER_EMAIL_RE = re.compile(rb'data-user-email-address="([^"]*)"') class NotParseable(Exception): @@ -787,6 +845,10 @@ class FormSpec: action_path: str page_count: int questions: list[Question] + #: Hidden inputs from the page, already HTML-unescaped. Carries `token` and + #: `tag`, both of which the POST is rejected without, and both of which + #: change on every page load. + hidden: dict[str, str] = field(default_factory=dict) @property def page_history(self) -> str: @@ -812,6 +874,43 @@ def extract_load_data(html: bytes) -> list: return json.loads(match.group(1)) +def requires_sign_in(html: bytes) -> bool: + """True when this page is a sign-in-required form viewed while signed out. + + Google Forms does not redirect in this case — it serves HTTP 200 with the + whole question structure present and the inputs disabled. Parsing therefore + succeeds on a signed-out page, so this check is the only thing standing + between a dead cookie and a wasted POST. + """ + return bool(_SIGN_IN_RE.search(html)) + + +def signed_in_email(html: bytes) -> str | None: + """The viewing account's address, or None when signed out. + + This is an HTML attribute Google populates from the session, so it is a far + stronger signal than pattern-matching the body for anything email-shaped. + """ + match = _VIEWER_EMAIL_RE.search(html) + if not match: + return None + value = match.group(1).decode("utf-8", "replace") + return value or None + + +def hidden_inputs(html: bytes) -> dict[str, str]: + """Every hidden form input, HTML-unescaped. + + Unescaping is not optional: `partialResponse` holds JSON whose quotes arrive + as `"`, and re-encoding the escaped form gets the POST rejected with a + 400 even when every field name is correct. + """ + return { + name.decode(): html_mod.unescape(value.decode()) + for name, value in _HIDDEN_RE.findall(html) + } + + def _items(load_data: list) -> list: try: return load_data[1][1] or [] @@ -857,6 +956,7 @@ def parse(html: bytes, url: str) -> FormSpec: action_path=f"/forms/d/e/{form_id}/formResponse", page_count=page_count, questions=questions, + hidden=hidden_inputs(html), ) ``` @@ -1158,7 +1258,19 @@ git commit -m "Add text-based answer matching with confidence scoring" - Produces: - `RequestBlob` dataclass: `host: str`, `path: str`, `body: str`, `headers: dict[str, str]` - `new_fbzx() -> str` - - `build(spec, filled, cookie, fbzx=None) -> RequestBlob` + - `build(spec, filled, cookie, email=None, fbzx=None) -> RequestBlob` + - `MissingToken` exception + +**Established empirically during the E1 gate — the payload the plan originally specified returns HTTP 400.** Measured against F1 by dropping one field at a time: + +| Field | Verdict | Source | +|---|---|---| +| `token` | **Required.** Without it: 400 | `spec.hidden["token"]`, rotates per page load | +| `tag` | **Required.** Without it: 400 | `spec.hidden["tag"]` | +| `emailAddress` | **Required** when the form collects the responder's email | caller supplies; `parse.signed_in_email()` reads it from the page | +| `partialResponse`, `dlut`, `hud` | Optional — dropping them still records | — | + +`token` is **not** a reCAPTCHA value: the page carries no recaptcha, grecaptcha, or botguard reference. It is a plain hidden input, which is why scraping it works and why the design survives. - [ ] **Step 1: Write the failing test** @@ -1173,10 +1285,11 @@ from fastform.match import FilledForm from fastform.parse import FormSpec -def spec(pages=1): +def spec(pages=1, hidden=None): return FormSpec( form_id="ABC", action_path="/forms/d/e/ABC/formResponse", page_count=pages, questions=[], + hidden=hidden if hidden is not None else {"token": "TOK", "tag": "TAG"}, ) @@ -1239,6 +1352,40 @@ class TestBuild(unittest.TestCase): str(len(blob.body.encode("utf-8"))), ) + def test_carries_token_and_tag_from_the_page(self): + # Measured: dropping either gets the POST rejected with 400. + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + fields = self._fields(blob) + self.assertEqual(fields["token"], "TOK") + self.assertEqual(fields["tag"], "TAG") + + def test_missing_token_raises_rather_than_posting_garbage(self): + with self.assertRaises(compile_mod.MissingToken): + compile_mod.build( + spec(hidden={"tag": "TAG"}), FilledForm(), "SID=x", fbzx="-1" + ) + + def test_missing_tag_raises(self): + with self.assertRaises(compile_mod.MissingToken): + compile_mod.build( + spec(hidden={"token": "TOK"}), FilledForm(), "SID=x", fbzx="-1" + ) + + def test_email_included_when_supplied(self): + blob = compile_mod.build( + spec(), FilledForm(), "SID=x", email="a@b.test", fbzx="-1" + ) + self.assertEqual(self._fields(blob)["emailAddress"], "a@b.test") + + def test_email_omitted_when_not_supplied(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + self.assertNotIn("emailAddress", self._fields(blob)) + + def test_page_fbzx_preferred_over_generated(self): + s = spec(hidden={"token": "TOK", "tag": "TAG", "fbzx": "42"}) + blob = compile_mod.build(s, FilledForm(), "SID=x") + self.assertEqual(self._fields(blob)["fbzx"], "42") + if __name__ == "__main__": unittest.main() @@ -1274,6 +1421,18 @@ from fastform.parse import FormSpec HOST = "docs.google.com" +#: Hidden inputs the server rejects the POST without. Established by dropping +#: each field in turn against a real form: omitting either yields HTTP 400. +REQUIRED_HIDDEN = ("token", "tag") + + +class MissingToken(Exception): + """The page did not yield `token`/`tag`, so no POST can succeed. + + Raised rather than sending a request that is certain to 400 — the failure is + better surfaced at arm time than as a mystery rejection mid-run. + """ + @dataclass class RequestBlob: @@ -1292,14 +1451,26 @@ def build( spec: FormSpec, filled: FilledForm, cookie: str, + email: str | None = None, fbzx: str | None = None, ) -> RequestBlob: + missing = [k for k in REQUIRED_HIDDEN if not spec.hidden.get(k)] + if missing: + raise MissingToken( + f"page yielded no {', '.join(missing)} — the POST would be rejected" + ) + fields: list[tuple[str, str]] = [ (f"entry.{entry_id}", value) for entry_id, value in filled.pairs ] + if email: + fields.append(("emailAddress", email)) fields.append(("fvv", "1")) - fields.append(("fbzx", fbzx or new_fbzx())) + # Prefer the page's own fbzx; it is already paired with this token. + fields.append(("fbzx", fbzx or spec.hidden.get("fbzx") or new_fbzx())) fields.append(("pageHistory", spec.page_history)) + for key in REQUIRED_HIDDEN: + fields.append((key, spec.hidden[key])) fields.append(("submissionTimestamp", "-1")) body = urllib.parse.urlencode(fields, encoding="utf-8") @@ -1752,6 +1923,25 @@ class TestClassifyStructural(unittest.TestCase): Outcome.AUTH_FAIL, ) + def test_signed_out_200_is_auth_fail_not_rejected(self): + """The regression this guards is the expensive one. + + Google serves a signed-out sign-in-required form as 200 with the full + question structure. Without the sign-in marker check this body reaches + the FB_PUBLIC_LOAD_DATA_ branch and reports REJECTED, sending the + operator to hunt a payload bug while the real problem is a dead cookie. + """ + body = ( + b'
' + b"" + ) + self.assertEqual(classify(200, None, body), Outcome.AUTH_FAIL) + + def test_400_with_form_rerendered_is_rejected(self): + # Measured: a payload missing a required field returns 400, not 200. + body = b"" + self.assertEqual(classify(400, None, body), Outcome.REJECTED) + def test_server_error_is_unknown(self): self.assertEqual(classify(500, None, b"oops"), Outcome.UNKNOWN) @@ -1782,6 +1972,12 @@ from enum import Enum #: the submission bounced and we are looking at the form again. FORM_DATA_MARKER = b"FB_PUBLIC_LOAD_DATA_" +#: Present only when a sign-in-required form is viewed while signed out. +#: Google serves that case as HTTP 200 with the full form structure rather than +#: redirecting, so without this marker a dead cookie is misread as a payload +#: rejection — the worst possible diagnosis to hand an operator mid-run. +SIGN_IN_MARKER = b'data-sign-in-to-continue="true"' + #: Present on the "your response has been recorded" page. CONFIRM_MARKERS = ( b"freebirdFormviewerViewResponseConfirmationMessage", @@ -1818,7 +2014,19 @@ def classify(status: int, location: str | None, body: bytes) -> Outcome: return Outcome.AUTH_FAIL return Outcome.UNKNOWN - if status != 200 or not body: + if not body: + return Outcome.UNKNOWN + + # Auth first. A signed-out response carries the full form structure, so any + # later check would claim it as a rejection. + if SIGN_IN_MARKER in body: + return Outcome.AUTH_FAIL + + # A rejected submission comes back 400 with the form re-rendered — measured, + # not assumed. Status alone therefore cannot decide anything. + if status == 400 and FORM_DATA_MARKER in body: + return Outcome.REJECTED + if status not in (200, 400): return Outcome.UNKNOWN # Order matters. Confirmation is checked before "closed" because a success @@ -2223,6 +2431,16 @@ def main(argv=None) -> int: path = parts.path + ("?" + parts.query if parts.query else "") page = conn.get(path, cookie) + # Auth check BEFORE parsing. A signed-out sign-in-required form still + # serves the full question structure at HTTP 200, so parsing would + # succeed and we would only learn the truth from a wasted POST. + if parse.requires_sign_in(page.body): + print("AUTH FAIL: the form loaded signed out.") + print("config/cookies.txt is stale — recapture it from DevTools.") + return int(ExitCode.AUTH) + + email = parse.signed_in_email(page.body) + try: spec = parse.parse(page.body, args.url) except parse.NotParseable as exc: @@ -2253,7 +2471,13 @@ def main(argv=None) -> int: print("\nAdd them to config/answers.toml, or re-run with --force.") return int(ExitCode.GAPS) - blob = compile_mod.build(spec, filled, cookie) + try: + blob = compile_mod.build(spec, filled, cookie, email=email) + except compile_mod.MissingToken as exc: + print(f"CANNOT BUILD REQUEST: {exc}") + print("Use the browser tab.") + return int(ExitCode.NOT_PARSEABLE) + result = conn.send(blob) outcome = classify_mod.classify(result.status, result.location, result.body) @@ -2261,6 +2485,7 @@ def main(argv=None) -> int: print(f"status {result.status}") print(f"elapsed {result.elapsed_ms:.1f} ms") print(f"confidence {filled.confidence:.2f}") + print(f"account {email or '(none detected)'}") if outcome is classify_mod.Outcome.UNKNOWN: print("\nSubmission status is UNDETERMINED. Check the response sheet.") diff --git a/docs/superpowers/specs/2026-08-05-fastform-design.md b/docs/superpowers/specs/2026-08-05-fastform-design.md index 1fa9680..77121e3 100644 --- a/docs/superpowers/specs/2026-08-05-fastform-design.md +++ b/docs/superpowers/specs/2026-08-05-fastform-design.md @@ -50,13 +50,32 @@ payload cannot be compiled. This forces a slower path (poll GET until it renders then arm, then POST — roughly two extra round trips) and no part of the design can avoid it. Acknowledged, not solved. -### No browser in the hot path - -Authentication needs cookies, not a browser. Cookies are extracted from Chrome -once, offline; the fire path is one stdlib HTTPS POST with a Cookie header. This -removes the largest dependency and eliminates a hard failure mode: -`launch_persistent_context` fails outright if Chrome is already running on the -target profile, which it will be on the day. +### No browser in the hot path, but cookies must be read live + +Authentication needs cookies, not a browser. The fire path is one stdlib HTTPS +POST with a Cookie header. This removes the largest dependency and eliminates a +hard failure mode: `launch_persistent_context` fails outright if Chrome is +already running on the target profile, which it will be on the day. + +**Cookies cannot be captured once and reused.** This was measured, not assumed: +a Cookie header copied from DevTools authenticated for about seven minutes and +was dead across all of Google within twenty. Chrome continuously rotates +`__Secure-1PSIDTS` — in the profile under test it had been refreshed four +minutes earlier, while `__Secure-1PSID` was thirty-five hours old — and once it +rotates, the captured value stops working. The browser session stays perfectly +healthy; only the snapshot dies. + +So the tool reads Chrome's cookie store at run time rather than replaying a +saved header. On macOS that is: the encryption secret from the login Keychain +(`security find-generic-password -s "Chrome Safe Storage"`), a key derived with +PBKDF2-HMAC-SHA1 over `saltysalt` at 1003 iterations, and AES-128-CBC over each +`v10`-prefixed value with a sixteen-space IV. Verified end to end with zero +Python dependencies, using an `openssl enc` subprocess for the cipher. + +Two consequences worth stating plainly. A boot-time cookie check hours before +the event proves nothing about the cookie that will exist at fire time. And +profile choice is a real decision, not a detail — the machine under test had two +signed-in Chrome profiles on different Google accounts. ### Connection warm-up is the largest deterministic win @@ -68,12 +87,30 @@ a fresh handshake when a send raises. ### Classify by structure, not by string -Google renders these pages in the account's UI language. Matching on -"Your response has been recorded" is fragile. Primary signals are structural: -did the response re-render a form (`FB_PUBLIC_LOAD_DATA_` present again → -rejected), did it land on the confirmation route, does a follow-up GET show the -already-responded state. Locale strings are a secondary signal, kept in a config -file populated during calibration. +Google renders these pages in the account's UI language, so string markers are +fragile. The structural signals below were measured against real responses, and +they replace an earlier guess that was wrong in a dangerous direction: + +| Signal | Meaning | +|---|---| +| `data-sign-in-to-continue="true"` | Signed out. Google serves this as HTTP 200 with the full form, never a redirect. | +| Response still contains `FastForm F1
FastForm F1
student@example.com Switch account
Email *
What is your full name? *
What is your email address?
 *
What is your phone number?
 *
Submit
Clear form
Never submit passwords through Google Forms.
This form was created outside of your domain. - Contact form owner - Terms of Service - Privacy Policy

Does this form look suspicious? Report

Google Forms
\ No newline at end of file diff --git a/fixtures/resp-rejected.bin b/fixtures/resp-rejected.bin new file mode 100644 index 0000000..4f4afa7 --- /dev/null +++ b/fixtures/resp-rejected.bin @@ -0,0 +1 @@ +FastForm F1
FastForm F1
student@example.com Switch account
Email *
What is your full name? *
What is your email address?
 *
What is your phone number?
 *
Submit
Clear form
Never submit passwords through Google Forms.
This form was created outside of your domain. - Contact form owner - Terms of Service - Privacy Policy

Does this form look suspicious? Report

Google Forms
\ No newline at end of file diff --git a/fixtures/resp-success.bin b/fixtures/resp-success.bin new file mode 100644 index 0000000..a92f95b --- /dev/null +++ b/fixtures/resp-success.bin @@ -0,0 +1 @@ +FastForm F1
FastForm F1
Your response has been recorded.
Submit another response
This form was created outside of your domain. - Contact form owner - Terms of Service - Privacy Policy

Does this form look suspicious? Report

Google Forms
\ No newline at end of file From dcff2c6a944eef7e1355f31325ea7031076a71f5 Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:05:30 +0700 Subject: [PATCH 11/24] Rewrite retry policy on measured evidence: no server-side idempotency Three protections the spec assumed were tested and all three are absent: fbzx deduplication - three identical POSTs sharing one fbzx, token and tag produced three separate rows. "Limit to 1 response" - with the setting on and prior responses from the same account, a direct POST appended a new row and changed nothing existing. ALREADY_RESPONDED - no such state on either verb. The GET returns an ordinary form; the POST returns an ordinary confirmation. The spec's check-then-act timeout policy therefore had nothing to check. It is replaced by: resend on timeout up to a low max_sends, since a duplicate row costs little in a first-three-wins race while a lost submission costs everything. The consequence is that the client's RECORDED hard-stop is now the only thing preventing duplicate entries, which makes it safety-critical. Optimistic POST-polling is downgraded to high-risk and off by default for the same reason. Also captured: a closed form 302s to /closedform and records nothing - the one genuine no-op available. And status 200 does not imply recorded; two probes returned 200 with no confirmation marker, so the classifier must never key on status alone. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- .../specs/2026-08-05-fastform-design.md | 66 ++++++++++++++----- fixtures/resp-already.bin | 1 + fixtures/resp-closed.bin | 1 + 3 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 fixtures/resp-already.bin create mode 100644 fixtures/resp-closed.bin diff --git a/docs/superpowers/specs/2026-08-05-fastform-design.md b/docs/superpowers/specs/2026-08-05-fastform-design.md index 77121e3..1fe0196 100644 --- a/docs/superpowers/specs/2026-08-05-fastform-design.md +++ b/docs/superpowers/specs/2026-08-05-fastform-design.md @@ -345,16 +345,25 @@ rather than guessed in config: re-validation shows the form has since closed — the organizer shut it to reset before the announced open. Because entry IDs are already in hand, POST-polling through the reopen saves the full round trip that `PRE_OPEN` cannot avoid. - Requires calibration (E4) confirming a closed-form POST is a no-op; without - that finding the mode is disabled and this case degrades to `PRE_OPEN` - behavior. + + **Now high-risk, and off by default.** It was gated on a closed-form POST + being a no-op, which holds — a closed form answers 302 to `/closedform` and + records nothing. But it also implicitly assumed some server-side duplicate + protection, and measurement found none: every accepted POST appends a row. A + single missed stop, or one ambiguous timeout mid-poll, writes duplicate + entries. Enable only with `max_sends` of 2 or 3 and only after rehearsing the + stop path; otherwise this case degrades to `PRE_OPEN` and costs one round trip. + + A further catch: `token` is per-page-load, so a POST-poll cannot reuse one + payload indefinitely. Token lifetime is unmeasured — treat it as a calibration + item before relying on this mode. Stop conditions: | Outcome | Action | |---|---| | `RECORDED` | Hard stop. No further sends under any circumstance. | -| `ALREADY_RESPONDED` | Stop. Treated as success. | +| `ALREADY_RESPONDED` | Stop. Treated as success. Measured absent on this form — retained only in case a differently configured form produces it. | | `REJECTED` | Stop. Payload is wrong; repeating will not help. Surface to gate. | | `CLOSED` | Keep polling — expected pre-open state. | | `AUTH_FAIL` | Stop loud, hand to browser. | @@ -363,20 +372,38 @@ Stop conditions: Plus a hard `max_sends` cap and a wall-clock deadline, both config. -**Timeout behavior branches on limit-to-one**, which `parse.py` already -established, so `strategy` branches on fact rather than a config guess: +**Timeout behaviour: resend, because nothing else is available.** This replaces +an earlier design that was built on a mechanism which turns out not to exist. + +Three protections were assumed and all three were measured absent: + +| Assumed protection | Measured reality | +|---|---| +| `fbzx` deduplicates repeats | No. Three identical POSTs sharing one `fbzx`, `token` and `tag` produced three separate rows. | +| "Limit to 1 response" blocks a second entry | No. With the setting on and prior responses from the same account, a direct POST appended a new row and changed nothing existing. | +| `ALREADY_RESPONDED` is observable | No such state on either verb. The GET returns an ordinary form; the POST returns an ordinary confirmation. | + +So there is **no server-side idempotency of any kind**, and no signal that +distinguishes "your send landed" from "your send was lost". The check-then-act +policy this spec originally specified has nothing to check. + +Given that, on a timeout the choice is between a possible duplicate row and a +possible missing submission. Resend, up to `max_sends`. In a first-three-wins +race the organizer takes the earliest qualifying entry, so a duplicate costs +approximately nothing while a lost submission costs everything. + +Two consequences worth stating plainly: -- **Limit-to-one ON** — GET-probe on a second warm socket. `ALREADY_RESPONDED` - means the send landed; stop. Anything else means it did not; resend. -- **Limit-to-one OFF** — the probe cannot distinguish recorded from lost, because - there is no already-responded state to observe. Resend directly, up to - `max_sends`. A duplicate row is acceptable here: the organizer takes the - earliest qualifying response, so an extra row costs nothing while a lost - submission costs everything. Skipping the probe also saves a round trip. +- **The client is the only safeguard.** `RECORDED` must hard-stop the loop, and + that stop is enforced by nothing but our own code. A bug there produces real + duplicate entries. +- **Duplicates are visible to the organizer.** Several identical rows under one + account may read as abuse even where the rules permit automation. Keep + `max_sends` low — 2 or 3, not 10. -`UNKNOWN` is deliberately more conservative than a timeout. A timeout means the -send may not have landed; an unclassifiable response means it may well have, and -we cannot tell what state we are in. +`UNKNOWN` remains more conservative than a timeout: a timeout means the send may +not have landed, while an unclassifiable response means it may well have. Stop +and surface it rather than resending blind. ### Jitter control @@ -431,8 +458,11 @@ Each emits fixtures and populates config. validation-rejected, closed, already-responded, and dead-cookies. These become `classify.py`'s unit tests. Record status codes, redirects, and whether `FB_PUBLIC_LOAD_DATA_` reappears. Note locale strings without depending on them. -- **E3 — overwrite semantics.** On F1, POST successfully then POST again: one - row or two, does the timestamp move? Repeat on F2. Gates optimistic mode. +- **E3 — overwrite semantics.** ANSWERED. Three identical POSTs sharing one + `fbzx`/`token`/`tag` produced three rows; with "Limit to 1 response" on, a + direct POST still appended a row and altered nothing existing. There is no + server-side deduplication and no overwrite. Re-run only to check whether this + changes. - **E4 — is a closed-form POST a no-op?** Toggle a form closed, POST, reopen, verify nothing from the closed window appears. Optimistic polling is unsafe without this. diff --git a/fixtures/resp-already.bin b/fixtures/resp-already.bin new file mode 100644 index 0000000..a38040c --- /dev/null +++ b/fixtures/resp-already.bin @@ -0,0 +1 @@ +FastForm F1
FastForm F1
Your response has been recorded.
This form was created outside of your domain. - Contact form owner - Terms of Service - Privacy Policy

Does this form look suspicious? Report

Google Forms
\ No newline at end of file diff --git a/fixtures/resp-closed.bin b/fixtures/resp-closed.bin new file mode 100644 index 0000000..24549ef --- /dev/null +++ b/fixtures/resp-closed.bin @@ -0,0 +1 @@ +FastForm F1
FastForm F1
The form FastForm F1 is no longer accepting responses.
Try contacting the owner of the form if you think this is a mistake.
This form was created outside of your domain. - Contact form owner - Terms of Service - Privacy Policy

Does this form look suspicious? Report

Google Forms
\ No newline at end of file From 32733e44e939e03084689a0c92537c8b347f4ed4 Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:05:59 +0700 Subject: [PATCH 12/24] Document response fixtures; drop mislabeled already-responded capture resp-already.bin held a confirmation page, not an already-responded page - that state does not exist, since limit-to-one does not block a direct POST. Keeping it under that name would have taught the classifier the wrong lesson. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- fixtures/README.md | 18 ++++++++++++++++++ fixtures/resp-already.bin | 1 - 2 files changed, 18 insertions(+), 1 deletion(-) delete mode 100644 fixtures/resp-already.bin diff --git a/fixtures/README.md b/fixtures/README.md index b743de5..f5180f1 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -8,3 +8,21 @@ Captured artifacts used as test data. Nothing here is generated by code. contains cookies. Response fixtures (`resp-*.bin`) are captured in Task 9. + +## Response fixtures + +Captured from real submissions to F1. These are ground truth for classify.py. + +- `resp-success.bin` — HTTP 200 confirmation page. Contains `FB_PUBLIC_LOAD_DATA_` + (so that marker proves nothing), no `
`, class `vHW8K` wrapping + "Your response has been recorded." +- `resp-rejected.bin` — HTTP 400, form re-rendered: ``, + `role="listitem"` x3, `name="token"` all present. + +Not captured, because the states do not exist: + +- **already-responded** — "Limit to 1 response" does not block a direct POST. + With it enabled and prior responses from the same account, the GET returns an + ordinary form and the POST records a new row. +- **closed** — a closed form answers 302 to `/closedform` before any body worth + keeping. Detection is the redirect target, not a page marker. diff --git a/fixtures/resp-already.bin b/fixtures/resp-already.bin deleted file mode 100644 index a38040c..0000000 --- a/fixtures/resp-already.bin +++ /dev/null @@ -1 +0,0 @@ -FastForm F1
FastForm F1
Your response has been recorded.
This form was created outside of your domain. - Contact form owner - Terms of Service - Privacy Policy

Does this form look suspicious? Report

Google Forms
\ No newline at end of file From 0379f7001f6bcd138849ca6a1c8b005d4f6acb3e Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:15:14 +0700 Subject: [PATCH 13/24] Read cookies live from Chrome's store --- fastform/cookies.py | 148 ++++++++++++++++++++++++++++++++++++++++++ tests/test_cookies.py | 78 ++++++++++++++++++++++ 2 files changed, 226 insertions(+) diff --git a/fastform/cookies.py b/fastform/cookies.py index 03b5317..84d1c76 100644 --- a/fastform/cookies.py +++ b/fastform/cookies.py @@ -5,8 +5,14 @@ months. See the plan's Scope Note. """ +import hashlib import http.client import re +import shutil +import sqlite3 +import subprocess +import tempfile +from dataclasses import dataclass from pathlib import Path USER_AGENT = ( @@ -61,3 +67,145 @@ def verify(cookie_header: str) -> tuple[bool, str | None]: match = _EMAIL_RE.search(body.decode("utf-8", "replace")) return True, match.group(0) if match else None + + +#: Chrome's macOS cookie encryption, all constants fixed by Chromium itself. +_SALT = b"saltysalt" +_ITERATIONS = 1003 +_KEY_LENGTH = 16 +_IV = b" " * 16 +_ENCRYPTED_PREFIXES = (b"v10", b"v11") + +#: Only these host_keys are sent to docs.google.com. Filtering here matters: +#: each value costs one openssl fork, and this cuts ~91 cookies down to ~24. +_GOOGLE_HOSTS = (".google.com", "docs.google.com", ".docs.google.com") + +CHROME_ROOT = Path.home() / "Library/Application Support/Google/Chrome" + + +class ChromeCookieError(Exception): + """Chrome's cookie store could not be read or decrypted.""" + + +@dataclass +class ChromeProfile: + name: str + path: Path + google_cookies: int + + +def keychain_secret() -> str: + """Chrome's encryption password, stored in the login Keychain. + + macOS may prompt for permission the first time. That prompt is the reason + this is worth doing well before the event rather than during it. + """ + proc = subprocess.run( + ["security", "find-generic-password", "-w", "-s", "Chrome Safe Storage"], + capture_output=True, text=True, timeout=30, + ) + if proc.returncode != 0: + raise ChromeCookieError( + f"could not read Chrome Safe Storage from Keychain: {proc.stderr.strip()}" + ) + return proc.stdout.strip() + + +def derive_key(secret: str) -> bytes: + return hashlib.pbkdf2_hmac( + "sha1", secret.encode(), _SALT, _ITERATIONS, _KEY_LENGTH + ) + + +def decrypt_value(blob: bytes, key: bytes) -> str | None: + """Decrypt one cookie value. Returns None for anything not v10/v11. + + Python has no stdlib AES, so this shells out to openssl rather than taking + a dependency. -nopad is required because we strip PKCS7 ourselves; letting + openssl do it makes a wrong key raise instead of returning garbage. + """ + if not blob or blob[:3] not in _ENCRYPTED_PREFIXES: + return None + proc = subprocess.run( + ["openssl", "enc", "-d", "-aes-128-cbc", + "-K", key.hex(), "-iv", _IV.hex(), "-nopad"], + input=blob[3:], capture_output=True, + ) + if proc.returncode != 0 or not proc.stdout: + return None + out = proc.stdout + pad = out[-1] + if 1 <= pad <= 16: + out = out[:-pad] + # Chrome >= v24 prefixes 32 bytes of domain hash before the value. + if len(out) > 32 and not re.fullmatch(rb"[\x20-\x7e]*", out[:32]): + out = out[32:] + return out.decode("utf-8", "replace") + + +def _read_cookies(db_path: Path) -> list[tuple[str, bytes]]: + """Copy first — Chrome holds a lock on the live database.""" + with tempfile.TemporaryDirectory() as tmp: + copy = Path(tmp) / "Cookies" + shutil.copy(db_path, copy) + con = sqlite3.connect(copy) + try: + placeholders = ",".join("?" * len(_GOOGLE_HOSTS)) + return con.execute( + f"SELECT name, encrypted_value FROM cookies " + f"WHERE host_key IN ({placeholders})", _GOOGLE_HOSTS + ).fetchall() + finally: + con.close() + + +def list_profiles(root: Path | None = None) -> list[ChromeProfile]: + """Every Chrome profile holding Google cookies, most cookies first. + + Profiles matter: a machine can have several signed into different accounts, + and the wrong one submits as the wrong person. + """ + root = root or CHROME_ROOT + if not root.is_dir(): + return [] + found = [] + for child in sorted(root.iterdir()): + db = child / "Cookies" + if not db.is_file(): + continue + try: + count = len(_read_cookies(db)) + except (sqlite3.Error, OSError): + continue + if count: + found.append(ChromeProfile(child.name, child, count)) + return sorted(found, key=lambda p: -p.google_cookies) + + +def cookie_header_from_chrome(profile: str | None = None) -> str: + """Build a Cookie header from Chrome's live store. + + Read at call time, never cached: Chrome rotates __Secure-1PSIDTS + continuously, and a header captured twenty minutes ago no longer + authenticates anywhere on Google. + """ + profiles = list_profiles() + if not profiles: + raise ChromeCookieError(f"no Chrome profile with Google cookies under {CHROME_ROOT}") + if profile: + chosen = next((p for p in profiles if p.name == profile), None) + if chosen is None: + names = ", ".join(p.name for p in profiles) + raise ChromeCookieError(f"no profile named {profile!r}; found: {names}") + else: + chosen = profiles[0] + + key = derive_key(keychain_secret()) + jar: dict[str, str] = {} + for name, blob in _read_cookies(chosen.path / "Cookies"): + value = decrypt_value(blob, key) + if value is not None: + jar.setdefault(name, value) + if not jar: + raise ChromeCookieError(f"decrypted no cookies from profile {chosen.name!r}") + return "; ".join(f"{k}={v}" for k, v in jar.items()) diff --git a/tests/test_cookies.py b/tests/test_cookies.py index 0a6b07a..a990cc1 100644 --- a/tests/test_cookies.py +++ b/tests/test_cookies.py @@ -1,3 +1,5 @@ +import sqlite3 +import subprocess import tempfile import unittest from pathlib import Path @@ -115,5 +117,81 @@ def test_connection_is_closed(self): conn.close.assert_called_once() +class TestDeriveKey(unittest.TestCase): + def test_key_is_16_bytes(self): + self.assertEqual(len(cookies.derive_key("abc")), 16) + + def test_deterministic(self): + self.assertEqual(cookies.derive_key("abc"), cookies.derive_key("abc")) + + def test_differs_per_secret(self): + self.assertNotEqual(cookies.derive_key("abc"), cookies.derive_key("abd")) + + +class TestDecryptValue(unittest.TestCase): + """Round-trip against openssl so the test proves the real cipher path.""" + + KEY = cookies.derive_key("test-secret") + + def _encrypt(self, plaintext: bytes) -> bytes: + proc = subprocess.run( + ["openssl", "enc", "-aes-128-cbc", "-K", self.KEY.hex(), + "-iv", (b" " * 16).hex()], + input=plaintext, capture_output=True, check=True, + ) + return b"v10" + proc.stdout + + def test_round_trip(self): + blob = self._encrypt(b"SID=hello-world") + self.assertEqual(cookies.decrypt_value(blob, self.KEY), "SID=hello-world") + + def test_unencrypted_blob_returns_none(self): + self.assertIsNone(cookies.decrypt_value(b"plaintext", self.KEY)) + + def test_empty_blob_returns_none(self): + self.assertIsNone(cookies.decrypt_value(b"", self.KEY)) + + def test_wrong_key_does_not_raise(self): + blob = self._encrypt(b"SID=hello-world") + other = cookies.derive_key("different") + result = cookies.decrypt_value(blob, other) + self.assertNotEqual(result, "SID=hello-world") + + +class TestListProfiles(unittest.TestCase): + def _profile_dir(self, name, rows): + root = Path(tempfile.mkdtemp()) + prof = root / name + prof.mkdir() + con = sqlite3.connect(prof / "Cookies") + con.execute( + "CREATE TABLE cookies (host_key TEXT, name TEXT, encrypted_value BLOB)" + ) + con.executemany("INSERT INTO cookies VALUES (?,?,?)", rows) + con.commit() + con.close() + return root + + def test_counts_google_cookies(self): + root = self._profile_dir( + "Profile 1", + [(".google.com", "SID", b"v10x"), ("example.com", "other", b"v10y")], + ) + profiles = cookies.list_profiles(root) + self.assertEqual(len(profiles), 1) + self.assertEqual(profiles[0].name, "Profile 1") + self.assertEqual(profiles[0].google_cookies, 1) + + def test_skips_profiles_without_a_cookie_db(self): + root = Path(tempfile.mkdtemp()) + (root / "Profile 9").mkdir() + self.assertEqual(cookies.list_profiles(root), []) + + def test_missing_root_returns_empty(self): + self.assertEqual( + cookies.list_profiles(Path(tempfile.mkdtemp()) / "nope"), [] + ) + + if __name__ == "__main__": unittest.main() From 2e72caf1f98e1cdeebe0ce31470c11d08b490522 Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:21:02 +0700 Subject: [PATCH 14/24] Require an explicit Chrome profile; add account resolver cookie_header_from_chrome() defaulted to the profile with the most Google cookies. Cookie count is not an identity, and a count shift on the day could silently switch the submitting account, so the profile argument is now required - the tool refuses to guess who submits. resolve_accounts() pairs each profile with its signed-in email via one authenticated GET each, so the operator chooses between accounts rather than between profile numbers. On the test machine only 2 of 5 profiles still authenticate. Also closes the reviewer's Minor: the 32-byte domain-hash-strip branch in decrypt_value now has a real openssl round-trip test, plus a guard that a sub-32-byte value is not mistaken for a prefixed one. And corrects the module docstring, which still claimed cookies "last months" - the assumption this whole line of work disproved. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- fastform/cookies.py | 70 +++++++++++++++++++++++++++++-------------- tests/test_cookies.py | 50 +++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 22 deletions(-) diff --git a/fastform/cookies.py b/fastform/cookies.py index 84d1c76..bba32f1 100644 --- a/fastform/cookies.py +++ b/fastform/cookies.py @@ -1,8 +1,10 @@ -"""Load a Cookie header captured from Chrome DevTools and check it still authenticates. +"""Google auth cookies: read them live from Chrome, or load a manual override. -Cookies are captured by hand rather than decrypted out of Chrome's SQLite -store: decryption needs AES (a dependency), and Google session cookies last -months. See the plan's Scope Note. +A Cookie header captured by hand dies within minutes — Chrome rotates +`__Secure-1PSIDTS` continuously, and the snapshot stops authenticating once it +does (measured: dead across all of Google within twenty minutes). So the real +source is `cookie_header_from_chrome()`, which decrypts Chrome's live store at +call time. `load()` remains only as a manual override for a hand-pasted header. """ import hashlib @@ -182,24 +184,7 @@ def list_profiles(root: Path | None = None) -> list[ChromeProfile]: return sorted(found, key=lambda p: -p.google_cookies) -def cookie_header_from_chrome(profile: str | None = None) -> str: - """Build a Cookie header from Chrome's live store. - - Read at call time, never cached: Chrome rotates __Secure-1PSIDTS - continuously, and a header captured twenty minutes ago no longer - authenticates anywhere on Google. - """ - profiles = list_profiles() - if not profiles: - raise ChromeCookieError(f"no Chrome profile with Google cookies under {CHROME_ROOT}") - if profile: - chosen = next((p for p in profiles if p.name == profile), None) - if chosen is None: - names = ", ".join(p.name for p in profiles) - raise ChromeCookieError(f"no profile named {profile!r}; found: {names}") - else: - chosen = profiles[0] - +def _decrypt_profile(chosen: ChromeProfile) -> str: key = derive_key(keychain_secret()) jar: dict[str, str] = {} for name, blob in _read_cookies(chosen.path / "Cookies"): @@ -209,3 +194,44 @@ def cookie_header_from_chrome(profile: str | None = None) -> str: if not jar: raise ChromeCookieError(f"decrypted no cookies from profile {chosen.name!r}") return "; ".join(f"{k}={v}" for k, v in jar.items()) + + +def cookie_header_from_chrome(profile: str) -> str: + """Build a Cookie header from a named Chrome profile's live store. + + `profile` is required, not optional. A machine can have several Chrome + profiles signed into different Google accounts, and cookie count — the only + thing distinguishing them without a network call — is not a basis for + choosing which identity submits. Pick with `resolve_accounts()` once, pin + the name in config, and pass it here. + + Read at call time, never cached: Chrome rotates __Secure-1PSIDTS + continuously, and a header captured twenty minutes ago no longer + authenticates anywhere on Google. + """ + profiles = list_profiles() + if not profiles: + raise ChromeCookieError(f"no Chrome profile with Google cookies under {CHROME_ROOT}") + chosen = next((p for p in profiles if p.name == profile), None) + if chosen is None: + names = ", ".join(p.name for p in profiles) + raise ChromeCookieError(f"no profile named {profile!r}; found: {names}") + return _decrypt_profile(chosen) + + +def resolve_accounts(root: Path | None = None) -> list[tuple[ChromeProfile, str | None]]: + """Pair each profile with the Google account it is signed into. + + A setup-time helper, not a hot-path call: it makes one authenticated GET per + profile. Cookie count tells the operator nothing about *which* account a + profile holds; this turns "Profile 6" into an email they can recognise. A + profile whose cookies no longer authenticate resolves to None. + """ + resolved = [] + for prof in list_profiles(root): + try: + _, email = verify(_decrypt_profile(prof)) + except ChromeCookieError: + email = None + resolved.append((prof, email)) + return resolved diff --git a/tests/test_cookies.py b/tests/test_cookies.py index a990cc1..3716a24 100644 --- a/tests/test_cookies.py +++ b/tests/test_cookies.py @@ -193,5 +193,55 @@ def test_missing_root_returns_empty(self): ) +class TestDecryptDomainHashPrefix(unittest.TestCase): + """Newer Chrome prefixes the plaintext with 32 bytes of domain hash. + + The reviewer flagged this branch as untested. A wrong strip corrupts every + cookie value while still looking plausible, so it is worth a real round-trip. + """ + + KEY = cookies.derive_key("hash-prefix-secret") + + def _encrypt(self, plaintext: bytes) -> bytes: + proc = subprocess.run( + ["openssl", "enc", "-aes-128-cbc", "-K", self.KEY.hex(), + "-iv", (b" " * 16).hex()], + input=plaintext, capture_output=True, check=True, + ) + return b"v10" + proc.stdout + + def test_32_byte_binary_prefix_is_stripped(self): + # A real domain hash is 32 non-printable bytes ahead of the value. + prefix = bytes(range(32)) + blob = self._encrypt(prefix + b"SID=value-after-hash") + self.assertEqual( + cookies.decrypt_value(blob, self.KEY), "SID=value-after-hash" + ) + + def test_short_printable_value_is_not_mistaken_for_a_prefix(self): + # A value under 32 bytes must survive intact — no prefix to strip. + blob = self._encrypt(b"SID=short") + self.assertEqual(cookies.decrypt_value(blob, self.KEY), "SID=short") + + +class TestCookieHeaderRequiresProfile(unittest.TestCase): + def test_profile_is_a_required_positional_argument(self): + with self.assertRaises(TypeError): + cookies.cookie_header_from_chrome() # noqa - intentionally missing arg + + def test_unknown_profile_raises_chrome_cookie_error(self): + real = cookies.list_profiles + + def fake(root=None): + return [cookies.ChromeProfile("Profile 1", Path("/x"), 5)] + + cookies.list_profiles = fake + try: + with self.assertRaises(cookies.ChromeCookieError): + cookies.cookie_header_from_chrome("Profile 99") + finally: + cookies.list_profiles = real + + if __name__ == "__main__": unittest.main() From 4780a1466eb88d00799c8451e34981c366dcb11c Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:23:12 +0700 Subject: [PATCH 15/24] Add profile.toml with Chrome profile setup, unset by default The competition account is not yet decided and may be neither of the two currently signed in, so the profile stays unset. The tool already refuses to run without it, so unset is a safe state. Documents how to list accounts and what to do if the right one is not signed into Chrome yet. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- config/profile.toml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 config/profile.toml diff --git a/config/profile.toml b/config/profile.toml new file mode 100644 index 0000000..a3f2b7a --- /dev/null +++ b/config/profile.toml @@ -0,0 +1,22 @@ +# Runtime configuration. + +[chrome] +# Which Chrome profile's cookies to submit with. REQUIRED before any real run — +# the tool refuses to guess, because picking the wrong profile submits as the +# wrong person. Leave unset until you know which Google account competes. +# +# To see the choices, run: +# python3 -c "from fastform import cookies; [print(f'{p.name}: {e}') for p,e in cookies.resolve_accounts()]" +# +# That prints each profile with the account it is signed into. Then set: +# profile = "Profile 1" +# +# Identified on this machine so far: +# Profile 1 -> student@example.com +# Profile 5 -> personal@example.com +# +# NEITHER is the competition account? Sign it into any Chrome profile first, +# then re-run the command above — a freshly signed-in profile shows up there +# and can be pinned like any other. + +# profile = From 64515db1c016ddf371439eb7c81f45334586c9e7 Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:26:23 +0700 Subject: [PATCH 16/24] Add synthetic F3/F4 parse fixtures to unblock Task 5 Real F3/F4 need browser work; these hand-built FB_PUBLIC_LOAD_DATA_ fixtures satisfy parse.py's multi-section and file-upload tests now. Verified against the plan's parse.py: F3 -> 3 pages/0,1,2/city options, F4 -> one file-upload blocker. Real forms remain a deferred fidelity check. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- fixtures/README.md | 8 ++++++++ fixtures/f3-viewform.html | 1 + fixtures/f4-viewform.html | 1 + 3 files changed, 10 insertions(+) create mode 100644 fixtures/f3-viewform.html create mode 100644 fixtures/f4-viewform.html diff --git a/fixtures/README.md b/fixtures/README.md index f5180f1..d7c3490 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -26,3 +26,11 @@ Not captured, because the states do not exist: ordinary form and the POST records a new row. - **closed** — a closed form answers 302 to `/closedform` before any body worth keeping. Detection is the redirect target, not a page marker. + +## Synthetic fixtures (f3, f4) + +`f3-viewform.html` and `f4-viewform.html` are SYNTHETIC, hand-built from the +FB_PUBLIC_LOAD_DATA_ structure verified against the real F1 page. They exist so +parse.py's multi-section and file-upload tests can run without creating real +F3/F4 Google Forms (human browser work). Building the real forms remains a +deferred fidelity check; the parser cannot tell the difference. diff --git a/fixtures/f3-viewform.html b/fixtures/f3-viewform.html new file mode 100644 index 0000000..de27e2d --- /dev/null +++ b/fixtures/f3-viewform.html @@ -0,0 +1 @@ +Synthetic \ No newline at end of file diff --git a/fixtures/f4-viewform.html b/fixtures/f4-viewform.html new file mode 100644 index 0000000..39e8b98 --- /dev/null +++ b/fixtures/f4-viewform.html @@ -0,0 +1 @@ +Synthetic \ No newline at end of file From 7bf98fa07510c17952704d61b9efb7b7d3532b5d Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:28:14 +0700 Subject: [PATCH 17/24] Add form structure parser Implements FB_PUBLIC_LOAD_DATA_ extraction, question and option parsing, sign-in detection, and hidden input scraping with HTML unescaping. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- fastform/parse.py | 174 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_parse.py | 138 +++++++++++++++++++++++++++++++++++ 2 files changed, 312 insertions(+) create mode 100644 fastform/parse.py create mode 100644 tests/test_parse.py diff --git a/fastform/parse.py b/fastform/parse.py new file mode 100644 index 0000000..89687f4 --- /dev/null +++ b/fastform/parse.py @@ -0,0 +1,174 @@ +"""Turn a Google Form page into a FormSpec. + +Everything comes out of the FB_PUBLIC_LOAD_DATA_ blob embedded in the page. +A closed form does not include that blob, which is why NotParseable exists and +why the caller has to treat it as a distinct state rather than an error. +""" + +import html as html_mod +import json +import re +from dataclasses import dataclass, field + +TYPE_SHORT_ANSWER = 0 +TYPE_PARAGRAPH = 1 +TYPE_MULTIPLE_CHOICE = 2 +TYPE_DROPDOWN = 3 +TYPE_CHECKBOXES = 4 +TYPE_LINEAR_SCALE = 5 +TYPE_GRID = 7 +TYPE_SECTION = 8 +TYPE_DATE = 9 +TYPE_TIME = 10 +TYPE_FILE_UPLOAD = 13 + +#: Types the direct-POST path cannot satisfy. Hitting one means browser handoff. +BLOCKER_TYPES = frozenset({TYPE_FILE_UPLOAD, TYPE_GRID}) + +#: Types whose value must match one of the offered options. +CHOICE_TYPES = frozenset({TYPE_MULTIPLE_CHOICE, TYPE_DROPDOWN, TYPE_CHECKBOXES}) + +_LOAD_DATA_RE = re.compile( + rb"FB_PUBLIC_LOAD_DATA_\s*=\s*(\[.*?\]);\s*", re.S +) +_FORM_ID_RE = re.compile(r"/forms/d/e/([^/]+)/") +_HIDDEN_RE = re.compile( + rb']+type="hidden"[^>]+name="([^"]+)"[^>]+value="([^"]*)"' +) +#: Present only when the viewer is signed out of a sign-in-required form. +_SIGN_IN_RE = re.compile(rb'data-sign-in-to-continue="true"') +_VIEWER_EMAIL_RE = re.compile(rb'data-user-email-address="([^"]*)"') + + +class NotParseable(Exception): + """The page carries no form structure — closed, deleted, or not a form.""" + + +@dataclass +class Question: + entry_id: str + text: str + type: int + required: bool + options: list[str] = field(default_factory=list) + + +@dataclass +class FormSpec: + form_id: str + action_path: str + page_count: int + questions: list[Question] + #: Hidden inputs from the page, already HTML-unescaped. Carries `token` and + #: `tag`, both of which the POST is rejected without, and both of which + #: change on every page load. + hidden: dict[str, str] = field(default_factory=dict) + + @property + def page_history(self) -> str: + """The pageHistory value that satisfies every section in one POST.""" + return ",".join(str(i) for i in range(self.page_count)) + + @property + def blockers(self) -> list[Question]: + return [q for q in self.questions if q.type in BLOCKER_TYPES] + + +def form_id_from_url(url: str) -> str: + match = _FORM_ID_RE.search(url) + if not match: + raise ValueError(f"not a published Google Form URL: {url}") + return match.group(1) + + +def extract_load_data(html: bytes) -> list: + match = _LOAD_DATA_RE.search(html) + if not match: + raise NotParseable("FB_PUBLIC_LOAD_DATA_ not found") + return json.loads(match.group(1)) + + +def requires_sign_in(html: bytes) -> bool: + """True when this page is a sign-in-required form viewed while signed out. + + Google Forms does not redirect in this case — it serves HTTP 200 with the + whole question structure present and the inputs disabled. Parsing therefore + succeeds on a signed-out page, so this check is the only thing standing + between a dead cookie and a wasted POST. + """ + return bool(_SIGN_IN_RE.search(html)) + + +def signed_in_email(html: bytes) -> str | None: + """The viewing account's address, or None when signed out. + + This is an HTML attribute Google populates from the session, so it is a far + stronger signal than pattern-matching the body for anything email-shaped. + """ + match = _VIEWER_EMAIL_RE.search(html) + if not match: + return None + value = match.group(1).decode("utf-8", "replace") + return value or None + + +def hidden_inputs(html: bytes) -> dict[str, str]: + """Every hidden form input, HTML-unescaped. + + Unescaping is not optional: `partialResponse` holds JSON whose quotes arrive + as `"`, and re-encoding the escaped form gets the POST rejected with a + 400 even when every field name is correct. + """ + return { + name.decode(): html_mod.unescape(value.decode()) + for name, value in _HIDDEN_RE.findall(html) + } + + +def _items(load_data: list) -> list: + try: + return load_data[1][1] or [] + except (IndexError, TypeError) as exc: + raise NotParseable(f"unexpected load data shape: {exc}") from exc + + +def _options(raw_options) -> list[str]: + out = [] + for opt in raw_options or []: + if opt and opt[0]: + out.append(str(opt[0])) + return out + + +def parse(html: bytes, url: str) -> FormSpec: + load_data = extract_load_data(html) + form_id = form_id_from_url(url) + + questions: list[Question] = [] + page_count = 1 + + for item in _items(load_data): + item_type = item[3] + if item_type == TYPE_SECTION: + page_count += 1 + continue + + title = item[1] or "" + for entry in item[4] or []: + questions.append( + Question( + entry_id=str(entry[0]), + text=title, + type=item_type, + required=bool(entry[2]), + options=_options(entry[1]), + ) + ) + + return FormSpec( + form_id=form_id, + action_path=f"/forms/d/e/{form_id}/formResponse", + page_count=page_count, + questions=questions, + hidden=hidden_inputs(html), + ) diff --git a/tests/test_parse.py b/tests/test_parse.py new file mode 100644 index 0000000..407c18d --- /dev/null +++ b/tests/test_parse.py @@ -0,0 +1,138 @@ +import unittest +from pathlib import Path + +from fastform import parse + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" + +F1_URL = "https://docs.google.com/forms/d/e/FAKEID123/viewform" + + +class TestFormIdFromUrl(unittest.TestCase): + def test_extracts_published_id(self): + self.assertEqual( + parse.form_id_from_url( + "https://docs.google.com/forms/d/e/1FAIpQLSabc/viewform?usp=sf_link" + ), + "1FAIpQLSabc", + ) + + def test_rejects_non_form_url(self): + with self.assertRaises(ValueError): + parse.form_id_from_url("https://example.com/hello") + + +class TestParseF1(unittest.TestCase): + @classmethod + def setUpClass(cls): + html = (FIXTURES / "f1-viewform.html").read_bytes() + cls.spec = parse.parse(html, F1_URL) + + def test_finds_three_questions(self): + self.assertEqual(len(self.spec.questions), 3) + + def test_question_text_is_readable(self): + texts = [q.text for q in self.spec.questions] + self.assertIn("What is your full name?", texts) + + def test_entry_ids_are_numeric_strings(self): + for q in self.spec.questions: + self.assertTrue(q.entry_id.isdigit(), q.entry_id) + + def test_all_required(self): + self.assertTrue(all(q.required for q in self.spec.questions)) + + def test_single_page(self): + self.assertEqual(self.spec.page_count, 1) + self.assertEqual(self.spec.page_history, "0") + + def test_action_path(self): + self.assertEqual( + self.spec.action_path, "/forms/d/e/FAKEID123/formResponse" + ) + + def test_no_blockers(self): + self.assertEqual(self.spec.blockers, []) + + def test_hidden_inputs_carry_token_and_tag(self): + self.assertIn("token", self.spec.hidden) + self.assertIn("tag", self.spec.hidden) + self.assertTrue(self.spec.hidden["token"]) + + def test_hidden_values_are_html_unescaped(self): + # partialResponse holds JSON, so it contains quotes that the page + # escapes as ". Leaving them escaped 400s the POST. + self.assertNotIn(""", self.spec.hidden.get("partialResponse", "")) + + +class TestSignInDetection(unittest.TestCase): + """Google Forms returns 200 for a signed-out sign-in-required form, with the + question structure fully present. Only an attribute distinguishes the two.""" + + SIGNED_OUT = b'
' + SIGNED_IN = b'
' + + def test_detects_signed_out(self): + self.assertTrue(parse.requires_sign_in(self.SIGNED_OUT)) + + def test_detects_signed_in(self): + self.assertFalse(parse.requires_sign_in(self.SIGNED_IN)) + + def test_extracts_email_when_signed_in(self): + self.assertEqual( + parse.signed_in_email(self.SIGNED_IN), "someone@example.com" + ) + + def test_no_email_when_signed_out(self): + self.assertIsNone(parse.signed_in_email(self.SIGNED_OUT)) + + +class TestHiddenInputs(unittest.TestCase): + def test_unescapes_values(self): + html = b'' + self.assertEqual(parse.hidden_inputs(html), {"x": 'a"b&c'}) + + def test_ignores_non_hidden_inputs(self): + html = b'' + self.assertEqual(parse.hidden_inputs(html), {}) + + +class TestParseF3(unittest.TestCase): + @classmethod + def setUpClass(cls): + html = (FIXTURES / "f3-viewform.html").read_bytes() + cls.spec = parse.parse(html, F1_URL) + + def test_three_pages(self): + self.assertEqual(self.spec.page_count, 3) + self.assertEqual(self.spec.page_history, "0,1,2") + + def test_choice_options_captured(self): + city = next(q for q in self.spec.questions if "city" in q.text.lower()) + self.assertIn("Bangkok", city.options) + self.assertIn("Chiang Mai", city.options) + + def test_optional_question_detected(self): + optional = [q for q in self.spec.questions if not q.required] + self.assertTrue(optional, "expected at least one optional question") + + +class TestParseF4(unittest.TestCase): + @classmethod + def setUpClass(cls): + html = (FIXTURES / "f4-viewform.html").read_bytes() + cls.spec = parse.parse(html, F1_URL) + + def test_file_upload_is_a_blocker(self): + self.assertEqual(len(self.spec.blockers), 1) + self.assertEqual(self.spec.blockers[0].type, parse.TYPE_FILE_UPLOAD) + + +class TestParseFailures(unittest.TestCase): + def test_missing_load_data_raises(self): + with self.assertRaises(parse.NotParseable): + parse.parse(b"form is closed", F1_URL) + + +if __name__ == "__main__": + unittest.main() From 002b0a572ce50c3f38fcdad9fbf0f63fbc48569e Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:32:34 +0700 Subject: [PATCH 18/24] Fix parse.parse to raise NotParseable on malformed items, not IndexError _items() already guards the top-level FB_PUBLIC_LOAD_DATA_ access, but the per-item unpacking loop in parse() (item[3], item[1], item[4], entry[0..2]) had no equivalent guard, so a structurally short item leaked a bare IndexError instead of the contracted NotParseable that callers rely on to distinguish "not a parseable form" from a real error. --- fastform/parse.py | 33 ++++++++++++++++++--------------- tests/test_parse.py | 13 +++++++++++++ 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/fastform/parse.py b/fastform/parse.py index 89687f4..e013cc4 100644 --- a/fastform/parse.py +++ b/fastform/parse.py @@ -148,22 +148,25 @@ def parse(html: bytes, url: str) -> FormSpec: page_count = 1 for item in _items(load_data): - item_type = item[3] - if item_type == TYPE_SECTION: - page_count += 1 - continue - - title = item[1] or "" - for entry in item[4] or []: - questions.append( - Question( - entry_id=str(entry[0]), - text=title, - type=item_type, - required=bool(entry[2]), - options=_options(entry[1]), + try: + item_type = item[3] + if item_type == TYPE_SECTION: + page_count += 1 + continue + + title = item[1] or "" + for entry in item[4] or []: + questions.append( + Question( + entry_id=str(entry[0]), + text=title, + type=item_type, + required=bool(entry[2]), + options=_options(entry[1]), + ) ) - ) + except (IndexError, TypeError, KeyError) as exc: + raise NotParseable(f"unexpected item shape: {exc}") from exc return FormSpec( form_id=form_id, diff --git a/tests/test_parse.py b/tests/test_parse.py index 407c18d..c1398a4 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -133,6 +133,19 @@ def test_missing_load_data_raises(self): with self.assertRaises(parse.NotParseable): parse.parse(b"form is closed", F1_URL) + def test_structurally_short_item_raises_not_parseable(self): + # item is [123, "t"] here -- far too short to unpack as + # [id, title, description, type, entries, ...]. The blob itself is + # present and well-formed, so this must not be confused with the + # "no FB_PUBLIC_LOAD_DATA_" case; it's a distinct malformed-item case + # that must still surface as NotParseable, not a bare IndexError. + html = ( + b'' + ) + with self.assertRaises(parse.NotParseable): + parse.parse(html, F1_URL) + if __name__ == "__main__": unittest.main() From 4d641953589dd300ce6de6789915bc15cf6e3629 Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:34:35 +0700 Subject: [PATCH 19/24] Add text-based answer matching with confidence scoring --- fastform/match.py | 109 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_match.py | 105 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 fastform/match.py create mode 100644 tests/test_match.py diff --git a/fastform/match.py b/fastform/match.py new file mode 100644 index 0000000..5da411b --- /dev/null +++ b/fastform/match.py @@ -0,0 +1,109 @@ +"""Match prepared answers to a form's questions by text, never by position. + +Position matching breaks the moment a question is added or reordered. Text +matching degrades gracefully: a near miss still scores, and the score is what +decides whether firing is safe or the operator has to look. +""" + +import difflib +import re +import tomllib +import unicodedata +from dataclasses import dataclass, field +from pathlib import Path + +from fastform.parse import BLOCKER_TYPES, CHOICE_TYPES, FormSpec, Question + +DEFAULT_BANK_PATH = Path(__file__).resolve().parent.parent / "config" / "answers.toml" + +#: Below this, a fuzzy hit is treated as no match at all. +FUZZY_FLOOR = 0.6 + +_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE) + + +@dataclass +class FilledForm: + pairs: list[tuple[str, str]] = field(default_factory=list) + gaps: list[Question] = field(default_factory=list) + blockers: list[Question] = field(default_factory=list) + confidence: float = 1.0 + + +def load_bank(path: Path | None = None) -> list[dict]: + path = path or DEFAULT_BANK_PATH + with path.open("rb") as fh: + return tomllib.load(fh).get("answer", []) + + +def normalize(text: str) -> str: + text = unicodedata.normalize("NFKC", text).casefold() + text = _PUNCT_RE.sub(" ", text) + return " ".join(text.split()) + + +def score_question(question: Question, entry: dict) -> float: + """How well does this bank entry match this question? 0.0 means no match.""" + pattern = entry.get("regex") + if pattern and re.search(pattern, question.text, re.IGNORECASE): + return 1.0 + + target = normalize(question.text) + target_tokens = set(target.split()) + best = 0.0 + + for phrase in entry.get("match", []): + candidate = normalize(phrase) + if not candidate: + continue + if candidate == target: + best = max(best, 0.95) + continue + if set(candidate.split()) <= target_tokens: + best = max(best, 0.8) + continue + ratio = difflib.SequenceMatcher(None, candidate, target).ratio() + if ratio >= FUZZY_FLOOR: + best = max(best, ratio) + + return best + + +def _resolve_choice(value: str, options: list[str]) -> str | None: + """A choice answer must be one of the offered options, or it will be rejected.""" + if not options: + return value + normalized = {normalize(o): o for o in options} + return normalized.get(normalize(value)) + + +def fill(spec: FormSpec, bank: list[dict]) -> FilledForm: + filled = FilledForm() + scores: list[float] = [] + + for question in spec.questions: + if question.type in BLOCKER_TYPES: + filled.blockers.append(question) + continue + + best_score, best_value = 0.0, None + for entry in bank: + score = score_question(question, entry) + if score > best_score: + best_score, best_value = score, entry["value"] + + if best_value is not None and question.type in CHOICE_TYPES: + best_value = _resolve_choice(best_value, question.options) + if best_value is None: + best_score = 0.0 + + if best_value is not None and best_score > 0.0: + filled.pairs.append((question.entry_id, best_value)) + if question.required: + scores.append(best_score) + elif question.required: + filled.gaps.append(question) + scores.append(0.0) + + filled.confidence = min(scores) if scores else 1.0 + return filled diff --git a/tests/test_match.py b/tests/test_match.py new file mode 100644 index 0000000..9c4be0e --- /dev/null +++ b/tests/test_match.py @@ -0,0 +1,105 @@ +import unittest + +from fastform import match +from fastform.parse import ( + TYPE_FILE_UPLOAD, + TYPE_MULTIPLE_CHOICE, + TYPE_SHORT_ANSWER, + FormSpec, + Question, +) + + +def spec(*questions): + return FormSpec( + form_id="X", action_path="/p", page_count=1, questions=list(questions) + ) + + +BANK = [ + {"match": ["full name", "your name"], "value": "Test Person"}, + {"match": ["email address"], "regex": "e-?mail", "value": "test@example.com"}, + {"match": ["which city"], "value": "Bangkok"}, +] + + +class TestNormalize(unittest.TestCase): + def test_casefolds_and_strips_punctuation(self): + self.assertEqual(match.normalize("What is your Name?"), "what is your name") + + def test_collapses_whitespace(self): + self.assertEqual(match.normalize("a b\n c"), "a b c") + + +class TestFill(unittest.TestCase): + def test_exact_text_match_fills(self): + s = spec(Question("111", "full name", TYPE_SHORT_ANSWER, True)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, [("111", "Test Person")]) + self.assertEqual(filled.gaps, []) + + def test_regex_wins_and_scores_one(self): + s = spec(Question("222", "Your E-Mail", TYPE_SHORT_ANSWER, True)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, [("222", "test@example.com")]) + self.assertEqual(filled.confidence, 1.0) + + def test_unmatched_required_becomes_a_gap(self): + s = spec(Question("333", "Favourite dinosaur", TYPE_SHORT_ANSWER, True)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, []) + self.assertEqual([q.entry_id for q in filled.gaps], ["333"]) + + def test_unmatched_optional_is_omitted_not_a_gap(self): + s = spec(Question("444", "Favourite dinosaur", TYPE_SHORT_ANSWER, False)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, []) + self.assertEqual(filled.gaps, []) + + def test_choice_value_must_exist_in_options(self): + s = spec( + Question( + "555", + "Which city?", + TYPE_MULTIPLE_CHOICE, + True, + options=["Bangkok Metropolitan Region", "Phuket"], + ) + ) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, []) + self.assertEqual([q.entry_id for q in filled.gaps], ["555"]) + + def test_choice_value_present_in_options_fills(self): + s = spec( + Question( + "666", + "Which city?", + TYPE_MULTIPLE_CHOICE, + True, + options=["Bangkok", "Phuket"], + ) + ) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, [("666", "Bangkok")]) + + def test_blockers_are_reported_separately(self): + s = spec(Question("777", "Upload", TYPE_FILE_UPLOAD, False)) + filled = match.fill(s, BANK) + self.assertEqual([q.entry_id for q in filled.blockers], ["777"]) + + def test_confidence_is_min_over_required_questions(self): + s = spec( + Question("111", "full name", TYPE_SHORT_ANSWER, True), + Question("333", "Favourite dinosaur", TYPE_SHORT_ANSWER, True), + ) + filled = match.fill(s, BANK) + self.assertEqual(filled.confidence, 0.0) + + def test_confidence_is_one_when_no_required_questions(self): + s = spec(Question("444", "Anything", TYPE_SHORT_ANSWER, False)) + self.assertEqual(match.fill(s, BANK).confidence, 1.0) + + +if __name__ == "__main__": + unittest.main() From d6fdbbb8fa6aa21b678a4a80ec9102c76a3ccfb4 Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:37:41 +0700 Subject: [PATCH 20/24] Add request compiler --- fastform/compile.py | 85 ++++++++++++++++++++++++++++++++ tests/test_compile.py | 112 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 fastform/compile.py create mode 100644 tests/test_compile.py diff --git a/fastform/compile.py b/fastform/compile.py new file mode 100644 index 0000000..c21f5f0 --- /dev/null +++ b/fastform/compile.py @@ -0,0 +1,85 @@ +"""Turn a filled form into a ready-to-send request. + +Everything expensive happens here, at arm time. Sending is then a socket write +with nothing left to compute — which is the whole point of separating this from +transport. +""" + +import random +import urllib.parse +from dataclasses import dataclass + +from fastform.cookies import USER_AGENT +from fastform.match import FilledForm +from fastform.parse import FormSpec + +HOST = "docs.google.com" + +#: Hidden inputs the server rejects the POST without. Established by dropping +#: each field in turn against a real form: omitting either yields HTTP 400. +REQUIRED_HIDDEN = ("token", "tag") + + +class MissingToken(Exception): + """The page did not yield `token`/`tag`, so no POST can succeed. + + Raised rather than sending a request that is certain to 400 — the failure is + better surfaced at arm time than as a mystery rejection mid-run. + """ + + +@dataclass +class RequestBlob: + host: str + path: str + body: str + headers: dict[str, str] + + +def new_fbzx() -> str: + """A fresh anti-duplicate token, in the same shape Google's own pages emit.""" + return str(random.randint(-(2**63), 2**63 - 1)) + + +def build( + spec: FormSpec, + filled: FilledForm, + cookie: str, + email: str | None = None, + fbzx: str | None = None, +) -> RequestBlob: + missing = [k for k in REQUIRED_HIDDEN if not spec.hidden.get(k)] + if missing: + raise MissingToken( + f"page yielded no {', '.join(missing)} — the POST would be rejected" + ) + + fields: list[tuple[str, str]] = [ + (f"entry.{entry_id}", value) for entry_id, value in filled.pairs + ] + if email: + fields.append(("emailAddress", email)) + fields.append(("fvv", "1")) + # Prefer the page's own fbzx; it is already paired with this token. + fields.append(("fbzx", fbzx or spec.hidden.get("fbzx") or new_fbzx())) + fields.append(("pageHistory", spec.page_history)) + for key in REQUIRED_HIDDEN: + fields.append((key, spec.hidden[key])) + fields.append(("submissionTimestamp", "-1")) + + body = urllib.parse.urlencode(fields, encoding="utf-8") + viewform = f"https://{HOST}/forms/d/e/{spec.form_id}/viewform" + + return RequestBlob( + host=HOST, + path=spec.action_path, + body=body, + headers={ + "Cookie": cookie, + "User-Agent": USER_AGENT, + "Content-Type": "application/x-www-form-urlencoded", + "Content-Length": str(len(body.encode("utf-8"))), + "Referer": viewform, + "Origin": f"https://{HOST}", + }, + ) diff --git a/tests/test_compile.py b/tests/test_compile.py new file mode 100644 index 0000000..8e1495d --- /dev/null +++ b/tests/test_compile.py @@ -0,0 +1,112 @@ +import unittest +import urllib.parse + +from fastform import compile as compile_mod +from fastform.match import FilledForm +from fastform.parse import FormSpec + + +def spec(pages=1, hidden=None): + return FormSpec( + form_id="ABC", action_path="/forms/d/e/ABC/formResponse", + page_count=pages, questions=[], + hidden=hidden if hidden is not None else {"token": "TOK", "tag": "TAG"}, + ) + + +class TestNewFbzx(unittest.TestCase): + def test_is_a_signed_integer_string(self): + value = compile_mod.new_fbzx() + int(value) + + def test_differs_between_calls(self): + self.assertNotEqual(compile_mod.new_fbzx(), compile_mod.new_fbzx()) + + +class TestBuild(unittest.TestCase): + def _fields(self, blob): + return dict(urllib.parse.parse_qsl(blob.body)) + + def test_entry_pairs_are_prefixed(self): + filled = FilledForm(pairs=[("123", "Alice")]) + blob = compile_mod.build(spec(), filled, "SID=x", fbzx="-1") + self.assertEqual(self._fields(blob)["entry.123"], "Alice") + + def test_includes_fvv_and_timestamp(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + fields = self._fields(blob) + self.assertEqual(fields["fvv"], "1") + self.assertEqual(fields["submissionTimestamp"], "-1") + + def test_page_history_covers_every_section(self): + blob = compile_mod.build(spec(pages=3), FilledForm(), "SID=x", fbzx="-1") + self.assertEqual(self._fields(blob)["pageHistory"], "0,1,2") + + def test_cookie_and_content_type_headers(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=abc", fbzx="-1") + self.assertEqual(blob.headers["Cookie"], "SID=abc") + self.assertEqual( + blob.headers["Content-Type"], "application/x-www-form-urlencoded" + ) + + def test_referer_points_at_the_viewform(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + self.assertEqual( + blob.headers["Referer"], + "https://docs.google.com/forms/d/e/ABC/viewform", + ) + + def test_unicode_values_survive_encoding(self): + filled = FilledForm(pairs=[("1", "กรุงเทพ")]) + blob = compile_mod.build(spec(), filled, "SID=x", fbzx="-1") + self.assertEqual(self._fields(blob)["entry.1"], "กรุงเทพ") + + def test_generates_fbzx_when_not_supplied(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x") + int(self._fields(blob)["fbzx"]) + + def test_content_length_matches_encoded_body(self): + filled = FilledForm(pairs=[("1", "กรุงเทพ")]) + blob = compile_mod.build(spec(), filled, "SID=x", fbzx="-1") + self.assertEqual( + blob.headers["Content-Length"], + str(len(blob.body.encode("utf-8"))), + ) + + def test_carries_token_and_tag_from_the_page(self): + # Measured: dropping either gets the POST rejected with 400. + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + fields = self._fields(blob) + self.assertEqual(fields["token"], "TOK") + self.assertEqual(fields["tag"], "TAG") + + def test_missing_token_raises_rather_than_posting_garbage(self): + with self.assertRaises(compile_mod.MissingToken): + compile_mod.build( + spec(hidden={"tag": "TAG"}), FilledForm(), "SID=x", fbzx="-1" + ) + + def test_missing_tag_raises(self): + with self.assertRaises(compile_mod.MissingToken): + compile_mod.build( + spec(hidden={"token": "TOK"}), FilledForm(), "SID=x", fbzx="-1" + ) + + def test_email_included_when_supplied(self): + blob = compile_mod.build( + spec(), FilledForm(), "SID=x", email="a@b.test", fbzx="-1" + ) + self.assertEqual(self._fields(blob)["emailAddress"], "a@b.test") + + def test_email_omitted_when_not_supplied(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + self.assertNotIn("emailAddress", self._fields(blob)) + + def test_page_fbzx_preferred_over_generated(self): + s = spec(hidden={"token": "TOK", "tag": "TAG", "fbzx": "42"}) + blob = compile_mod.build(s, FilledForm(), "SID=x") + self.assertEqual(self._fields(blob)["fbzx"], "42") + + +if __name__ == "__main__": + unittest.main() From dfd8e48407ae660560ba22c8a8e43e73b31f6588 Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:42:05 +0700 Subject: [PATCH 21/24] Rewrite Task 9 classifier from measured fixtures The plan's classifier still had the investigation's disproven logic: it keyed success on FB_PUBLIC_LOAD_DATA_ reappearing (present on the success page too) and on freebird* classes that do not exist, so it reported RECORDED submissions as REJECTED. Rewritten to key on the element (primary recorded-vs-rejected split), the /closedform redirect, and the sign-in attribute. Verified 11/11 against the three real captured fixtures plus redirect and signed-out edge cases. Dropped the resp-already.bin references - that state does not exist. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- .../plans/2026-08-05-fastform-phases-0-4.md | 486 +++++++++++++++--- 1 file changed, 411 insertions(+), 75 deletions(-) diff --git a/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md b/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md index c463ed5..165b24f 100644 --- a/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md +++ b/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md @@ -354,6 +354,324 @@ git commit -m "Add cookie loading and freshness check" --- +### Task 2b: Read cookies live from Chrome's cookie store + +**Files:** +- Modify: `fastform/cookies.py` +- Modify: `tests/test_cookies.py` + +**Interfaces:** +- Consumes: nothing new. +- Produces: + - `ChromeProfile` dataclass: `name: str`, `path: Path`, `google_cookies: int` + - `list_profiles() -> list[ChromeProfile]` + - `keychain_secret() -> str` + - `derive_key(secret: str) -> bytes` + - `decrypt_value(blob: bytes, key: bytes) -> str | None` + - `cookie_header_from_chrome(profile: str | None = None) -> str` + - `ChromeCookieError` exception + - existing `load()`, `verify()`, `USER_AGENT` stay as they are + +**Why this task exists.** Task 2 reads a Cookie header captured by hand. Measurement showed such a header authenticates for roughly seven minutes and is dead across all of Google within twenty: Chrome continuously rotates `__Secure-1PSIDTS`, and once it does, the captured value stops working. A static header cannot support a boot-time check hours early, cannot support `ARMED_HOLDING`, and barely supports a hand-run tool. `load()` survives only as a manual override. + +**Every mechanism below was verified end to end on this machine** — 91/91 Google cookies decrypted, and the resulting header authenticated against a real form. Zero Python dependencies. + +- Keychain secret: `security find-generic-password -w -s "Chrome Safe Storage"` +- Key: `PBKDF2-HMAC-SHA1(secret, b"saltysalt", 1003, dklen=16)` — `hashlib`, stdlib +- Cipher: AES-128-CBC, IV of sixteen spaces, values prefixed `v10`/`v11`. No stdlib AES, so shell out to `openssl enc -d -aes-128-cbc -K -iv -nopad` +- Strip PKCS7 padding, then strip a 32-byte domain-hash prefix if present (newer Chrome adds one; detect by the first 32 bytes not being printable ASCII) +- Chrome holds a lock on the live DB — copy it before opening +- Filter `host_key IN ('.google.com', 'docs.google.com', '.docs.google.com')`; this machine had 91 Google cookies but only 24 relevant ones, and each decrypt is a subprocess fork, so filtering first roughly quarters the cost + +**Profiles matter.** This machine has two signed-in Chrome profiles on different Google accounts (`Profile 1` and `Profile 5`). Picking the wrong one submits as the wrong person. `list_profiles()` exists so the operator can see the choice. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_cookies.py`: + +```python +import sqlite3 +import subprocess +import tempfile +import unittest +from pathlib import Path + +from fastform import cookies + + +class TestDeriveKey(unittest.TestCase): + def test_key_is_16_bytes(self): + self.assertEqual(len(cookies.derive_key("abc")), 16) + + def test_deterministic(self): + self.assertEqual(cookies.derive_key("abc"), cookies.derive_key("abc")) + + def test_differs_per_secret(self): + self.assertNotEqual(cookies.derive_key("abc"), cookies.derive_key("abd")) + + +class TestDecryptValue(unittest.TestCase): + """Round-trip against openssl so the test proves the real cipher path.""" + + KEY = cookies.derive_key("test-secret") + + def _encrypt(self, plaintext: bytes) -> bytes: + proc = subprocess.run( + ["openssl", "enc", "-aes-128-cbc", "-K", self.KEY.hex(), + "-iv", (b" " * 16).hex()], + input=plaintext, capture_output=True, check=True, + ) + return b"v10" + proc.stdout + + def test_round_trip(self): + blob = self._encrypt(b"SID=hello-world") + self.assertEqual(cookies.decrypt_value(blob, self.KEY), "SID=hello-world") + + def test_unencrypted_blob_returns_none(self): + self.assertIsNone(cookies.decrypt_value(b"plaintext", self.KEY)) + + def test_empty_blob_returns_none(self): + self.assertIsNone(cookies.decrypt_value(b"", self.KEY)) + + def test_wrong_key_does_not_raise(self): + blob = self._encrypt(b"SID=hello-world") + other = cookies.derive_key("different") + result = cookies.decrypt_value(blob, other) + self.assertNotEqual(result, "SID=hello-world") + + +class TestListProfiles(unittest.TestCase): + def _profile_dir(self, name, rows): + root = Path(tempfile.mkdtemp()) + prof = root / name + prof.mkdir() + con = sqlite3.connect(prof / "Cookies") + con.execute( + "CREATE TABLE cookies (host_key TEXT, name TEXT, encrypted_value BLOB)" + ) + con.executemany("INSERT INTO cookies VALUES (?,?,?)", rows) + con.commit() + con.close() + return root + + def test_counts_google_cookies(self): + root = self._profile_dir( + "Profile 1", + [(".google.com", "SID", b"v10x"), ("example.com", "other", b"v10y")], + ) + profiles = cookies.list_profiles(root) + self.assertEqual(len(profiles), 1) + self.assertEqual(profiles[0].name, "Profile 1") + self.assertEqual(profiles[0].google_cookies, 1) + + def test_skips_profiles_without_a_cookie_db(self): + root = Path(tempfile.mkdtemp()) + (root / "Profile 9").mkdir() + self.assertEqual(cookies.list_profiles(root), []) + + def test_missing_root_returns_empty(self): + self.assertEqual( + cookies.list_profiles(Path(tempfile.mkdtemp()) / "nope"), [] + ) +``` + +- [ ] **Step 2: Run to verify they fail** + +```bash +cd /Users/supa/projects/active/fastform +python3 -m unittest tests.test_cookies -v +``` + +Expected: `AttributeError: module 'fastform.cookies' has no attribute 'derive_key'` + +- [ ] **Step 3: Write the implementation** + +Append to `fastform/cookies.py`: + +```python +import hashlib +import shutil +import sqlite3 +import subprocess +import tempfile +from dataclasses import dataclass + +#: Chrome's macOS cookie encryption, all constants fixed by Chromium itself. +_SALT = b"saltysalt" +_ITERATIONS = 1003 +_KEY_LENGTH = 16 +_IV = b" " * 16 +_ENCRYPTED_PREFIXES = (b"v10", b"v11") + +#: Only these host_keys are sent to docs.google.com. Filtering here matters: +#: each value costs one openssl fork, and this cuts ~91 cookies down to ~24. +_GOOGLE_HOSTS = (".google.com", "docs.google.com", ".docs.google.com") + +CHROME_ROOT = Path.home() / "Library/Application Support/Google/Chrome" + + +class ChromeCookieError(Exception): + """Chrome's cookie store could not be read or decrypted.""" + + +@dataclass +class ChromeProfile: + name: str + path: Path + google_cookies: int + + +def keychain_secret() -> str: + """Chrome's encryption password, stored in the login Keychain. + + macOS may prompt for permission the first time. That prompt is the reason + this is worth doing well before the event rather than during it. + """ + proc = subprocess.run( + ["security", "find-generic-password", "-w", "-s", "Chrome Safe Storage"], + capture_output=True, text=True, timeout=30, + ) + if proc.returncode != 0: + raise ChromeCookieError( + f"could not read Chrome Safe Storage from Keychain: {proc.stderr.strip()}" + ) + return proc.stdout.strip() + + +def derive_key(secret: str) -> bytes: + return hashlib.pbkdf2_hmac( + "sha1", secret.encode(), _SALT, _ITERATIONS, _KEY_LENGTH + ) + + +def decrypt_value(blob: bytes, key: bytes) -> str | None: + """Decrypt one cookie value. Returns None for anything not v10/v11. + + Python has no stdlib AES, so this shells out to openssl rather than taking + a dependency. -nopad is required because we strip PKCS7 ourselves; letting + openssl do it makes a wrong key raise instead of returning garbage. + """ + if not blob or blob[:3] not in _ENCRYPTED_PREFIXES: + return None + proc = subprocess.run( + ["openssl", "enc", "-d", "-aes-128-cbc", + "-K", key.hex(), "-iv", _IV.hex(), "-nopad"], + input=blob[3:], capture_output=True, + ) + if proc.returncode != 0 or not proc.stdout: + return None + out = proc.stdout + pad = out[-1] + if 1 <= pad <= 16: + out = out[:-pad] + # Chrome >= v24 prefixes 32 bytes of domain hash before the value. + if len(out) > 32 and not re.fullmatch(rb"[\x20-\x7e]*", out[:32]): + out = out[32:] + return out.decode("utf-8", "replace") + + +def _read_cookies(db_path: Path) -> list[tuple[str, bytes]]: + """Copy first — Chrome holds a lock on the live database.""" + with tempfile.TemporaryDirectory() as tmp: + copy = Path(tmp) / "Cookies" + shutil.copy(db_path, copy) + con = sqlite3.connect(copy) + try: + placeholders = ",".join("?" * len(_GOOGLE_HOSTS)) + return con.execute( + f"SELECT name, encrypted_value FROM cookies " + f"WHERE host_key IN ({placeholders})", _GOOGLE_HOSTS + ).fetchall() + finally: + con.close() + + +def list_profiles(root: Path | None = None) -> list[ChromeProfile]: + """Every Chrome profile holding Google cookies, most cookies first. + + Profiles matter: a machine can have several signed into different accounts, + and the wrong one submits as the wrong person. + """ + root = root or CHROME_ROOT + if not root.is_dir(): + return [] + found = [] + for child in sorted(root.iterdir()): + db = child / "Cookies" + if not db.is_file(): + continue + try: + count = len(_read_cookies(db)) + except (sqlite3.Error, OSError): + continue + if count: + found.append(ChromeProfile(child.name, child, count)) + return sorted(found, key=lambda p: -p.google_cookies) + + +def cookie_header_from_chrome(profile: str | None = None) -> str: + """Build a Cookie header from Chrome's live store. + + Read at call time, never cached: Chrome rotates __Secure-1PSIDTS + continuously, and a header captured twenty minutes ago no longer + authenticates anywhere on Google. + """ + profiles = list_profiles() + if not profiles: + raise ChromeCookieError(f"no Chrome profile with Google cookies under {CHROME_ROOT}") + if profile: + chosen = next((p for p in profiles if p.name == profile), None) + if chosen is None: + names = ", ".join(p.name for p in profiles) + raise ChromeCookieError(f"no profile named {profile!r}; found: {names}") + else: + chosen = profiles[0] + + key = derive_key(keychain_secret()) + jar: dict[str, str] = {} + for name, blob in _read_cookies(chosen.path / "Cookies"): + value = decrypt_value(blob, key) + if value is not None: + jar.setdefault(name, value) + if not jar: + raise ChromeCookieError(f"decrypted no cookies from profile {chosen.name!r}") + return "; ".join(f"{k}={v}" for k, v in jar.items()) +``` + +- [ ] **Step 4: Run the tests** + +```bash +python3 -m unittest tests.test_cookies -v +``` + +Expected: all tests pass, old and new. + +- [ ] **Step 5: Verify against the real Chrome store** + +```bash +python3 -c " +from fastform import cookies +for p in cookies.list_profiles(): + print(f'{p.name:12} {p.google_cookies} google cookies') +h = cookies.cookie_header_from_chrome() +print(f'header: {len(h)} chars, {h.count(\"=\")} cookies') +print('authenticates:', cookies.verify(h)) +" +``` + +Expected: at least one profile listed, a header of a couple of thousand characters, and `verify` returning `(True, ...)`. + +macOS may show a Keychain prompt on first run — approve it. That prompt is exactly why this must be exercised well before the event. + +- [ ] **Step 6: Commit** + +```bash +git add fastform/cookies.py tests/test_cookies.py +git commit -m "Read cookies live from Chrome's store" +``` + +--- + ### Task 3: E1 gate — prove a cookie-only POST records a response **Files:** @@ -1897,50 +2215,66 @@ from fastform.classify import Outcome, classify FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" -# Captured in Task 9 Step 2. The first two are POST responses; the last two are -# GET bodies, because a closed or already-responded form never renders far -# enough to POST. classify() handles both, which is why this list is uniform. -CASES = [ - ("resp-success.bin", 200, None, Outcome.RECORDED), - ("resp-rejected.bin", 200, None, Outcome.REJECTED), - ("resp-already.bin", 200, None, Outcome.ALREADY_RESPONDED), - ("resp-closed.bin", 200, None, Outcome.CLOSED), -] +class TestClassifyRealFixtures(unittest.TestCase): + """Ground truth: bodies captured from real submissions to F1. -class TestClassifyFixtures(unittest.TestCase): - def test_each_fixture_maps_to_its_outcome(self): - for name, status, location, expected in CASES: - with self.subTest(fixture=name): - body = (FIXTURES / name).read_bytes() - self.assertEqual(classify(status, location, body), expected) + resp-success.bin is the case an earlier classifier got wrong (reported + REJECTED on a recorded submission), so it is the most important row here. + """ + def test_success_is_recorded(self): + body = (FIXTURES / "resp-success.bin").read_bytes() + self.assertEqual(classify(200, None, body), Outcome.RECORDED) -class TestClassifyStructural(unittest.TestCase): - def test_redirect_to_accounts_is_auth_fail(self): + def test_rejected_is_rejected(self): + body = (FIXTURES / "resp-rejected.bin").read_bytes() + self.assertEqual(classify(400, None, body), Outcome.REJECTED) + + def test_closed_landing_page_is_closed(self): + body = (FIXTURES / "resp-closed.bin").read_bytes() + self.assertEqual(classify(200, None, body), Outcome.CLOSED) + + +class TestClassifyRedirects(unittest.TestCase): + def test_302_to_closedform_is_closed(self): + self.assertEqual( + classify(302, "https://docs.google.com/forms/u/0/d/e/X/closedform", b""), + Outcome.CLOSED, + ) + + def test_302_to_accounts_is_auth_fail(self): self.assertEqual( - classify(302, "https://accounts.google.com/signin", b""), + classify(302, "https://accounts.google.com/ServiceLogin", b""), Outcome.AUTH_FAIL, ) - def test_signed_out_200_is_auth_fail_not_rejected(self): - """The regression this guards is the expensive one. + def test_other_redirect_is_unknown(self): + self.assertEqual( + classify(302, "https://docs.google.com/forms/d/e/X/viewform", b""), + Outcome.UNKNOWN, + ) - Google serves a signed-out sign-in-required form as 200 with the full - question structure. Without the sign-in marker check this body reaches - the FB_PUBLIC_LOAD_DATA_ branch and reports REJECTED, sending the - operator to hunt a payload bug while the real problem is a dead cookie. - """ + +class TestClassifyStructural(unittest.TestCase): + def test_signed_out_200_is_auth_fail_not_rejected(self): + # A signed-out sign-in-required form is served as 200 with the whole + # form. Without the sign-in check this reaches the form-element branch + # and reports REJECTED, sending the operator after a phantom payload bug. body = ( b'
' b"" + b'' ) self.assertEqual(classify(200, None, body), Outcome.AUTH_FAIL) - def test_400_with_form_rerendered_is_rejected(self): - # Measured: a payload missing a required field returns 400, not 200. - body = b"" - self.assertEqual(classify(400, None, body), Outcome.REJECTED) + def test_form_element_on_200_is_rejected(self): + body = b'' + self.assertEqual(classify(200, None, body), Outcome.REJECTED) + + def test_terminal_page_without_markers_is_unknown(self): + # No form, but no recorded marker either: never guess RECORDED. + self.assertEqual(classify(200, None, b"something else"), Outcome.UNKNOWN) def test_server_error_is_unknown(self): self.assertEqual(classify(500, None, b"oops"), Outcome.UNKNOWN) @@ -1958,40 +2292,44 @@ if __name__ == "__main__": Create `fastform/classify.py`: ```python -"""Decide what a response means. - -Structural signals first, language-dependent strings only as a fallback: these -pages render in the account's UI language, so an English marker is not something -to rely on. The markers below were derived from real fixtures — see -tests/test_classify.py. +"""Decide what a response means, from measured fixtures — not guesses. + +Every rule here was derived from real captured responses (see fixtures/README). +The signals are structural and locale-independent, which matters because the +account under test renders Forms in Thai. An earlier version keyed on +`FB_PUBLIC_LOAD_DATA_` reappearing and on `freebird*` CSS classes; both were +wrong — the success page also carries `FB_PUBLIC_LOAD_DATA_`, and those classes +do not exist in current markup. That mistake reported a recorded submission as +REJECTED, which is the worst error available because it invites a retry. """ from enum import Enum +from urllib.parse import urlsplit -#: Present whenever a live form is rendered. Its reappearance after a POST means -#: the submission bounced and we are looking at the form again. -FORM_DATA_MARKER = b"FB_PUBLIC_LOAD_DATA_" +#: The actual element. Present only when the form came back for +#: another attempt, i.e. the submission did NOT record. This is the primary +#: recorded-vs-rejected discriminator. +FORM_ELEMENT = b" bool: - lowered = body.lower() - return any(marker.lower() in lowered for marker in markers) + return any(marker in body for marker in markers) def classify(status: int, location: str | None, body: bytes) -> Outcome: + # Redirects carry the signal before any body is read. A closed form 302s to + # /closedform; a dead session 302s to accounts.google.com. if status in (301, 302, 303, 307, 308): - if location and "accounts.google.com" in location: + loc = location or "" + if "accounts.google.com" in loc: return Outcome.AUTH_FAIL + if urlsplit(loc).path.endswith("/closedform"): + return Outcome.CLOSED return Outcome.UNKNOWN if not body: return Outcome.UNKNOWN - # Auth first. A signed-out response carries the full form structure, so any - # later check would claim it as a rejection. + # Signed-out sign-in-required form: HTTP 200 with the whole form present. + # Must precede the form-element test below. if SIGN_IN_MARKER in body: return Outcome.AUTH_FAIL - # A rejected submission comes back 400 with the form re-rendered — measured, - # not assumed. Status alone therefore cannot decide anything. - if status == 400 and FORM_DATA_MARKER in body: + # The closed landing page, if a redirect was followed through to it. + if CLOSED_MARKER in body and LOAD_DATA not in body: + return Outcome.CLOSED + + # A re-rendered form means the submission did not take. Measured rejects come + # back 400, but keying on the form element as well as the status is stronger. + if status == 400 or FORM_ELEMENT in body: return Outcome.REJECTED - if status not in (200, 400): - return Outcome.UNKNOWN - # Order matters. Confirmation is checked before "closed" because a success - # page also lacks FB_PUBLIC_LOAD_DATA_, and a loose closed-marker would - # otherwise swallow it. - if _has_any(body, ALREADY_MARKERS): - return Outcome.ALREADY_RESPONDED - if _has_any(body, CONFIRM_MARKERS): + # Terminal confirmation page: no form element, positive recorded marker. + if _has_any(body, RECORDED_MARKERS): return Outcome.RECORDED - if _has_any(body, CLOSED_MARKERS): - return Outcome.CLOSED - if FORM_DATA_MARKER in body: - return Outcome.REJECTED return Outcome.UNKNOWN ``` From 843f650d71c53d06ec5e24fb14e8aca8634eafbb Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:43:54 +0700 Subject: [PATCH 22/24] Add response classifier (Task 9) Implement Outcome enum (RECORDED, REJECTED, CLOSED, ALREADY_RESPONDED, AUTH_FAIL, UNKNOWN) and classify() function to map HTTP responses to outcomes using real captured fixtures. Classifier detects: redirect status/location, form re-render, sign-in requirement, closed-form page, and positive confirmation markers. All 11 classify tests pass; full suite 86/86 passing. --- fastform/classify.py | 86 ++++++++++++++++++++++++++++++++++++++++++ tests/test_classify.py | 77 +++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 fastform/classify.py create mode 100644 tests/test_classify.py diff --git a/fastform/classify.py b/fastform/classify.py new file mode 100644 index 0000000..ca9b40b --- /dev/null +++ b/fastform/classify.py @@ -0,0 +1,86 @@ +"""Decide what a response means, from measured fixtures — not guesses. + +Every rule here was derived from real captured responses (see fixtures/README). +The signals are structural and locale-independent, which matters because the +account under test renders Forms in Thai. An earlier version keyed on +`FB_PUBLIC_LOAD_DATA_` reappearing and on `freebird*` CSS classes; both were +wrong — the success page also carries `FB_PUBLIC_LOAD_DATA_`, and those classes +do not exist in current markup. That mistake reported a recorded submission as +REJECTED, which is the worst error available because it invites a retry. +""" + +from enum import Enum +from urllib.parse import urlsplit + +#: The actual element. Present only when the form came back for +#: another attempt, i.e. the submission did NOT record. This is the primary +#: recorded-vs-rejected discriminator. +FORM_ELEMENT = b" bool: + return any(marker in body for marker in markers) + + +def classify(status: int, location: str | None, body: bytes) -> Outcome: + # Redirects carry the signal before any body is read. A closed form 302s to + # /closedform; a dead session 302s to accounts.google.com. + if status in (301, 302, 303, 307, 308): + loc = location or "" + if "accounts.google.com" in loc: + return Outcome.AUTH_FAIL + if urlsplit(loc).path.endswith("/closedform"): + return Outcome.CLOSED + return Outcome.UNKNOWN + + if not body: + return Outcome.UNKNOWN + + # Signed-out sign-in-required form: HTTP 200 with the whole form present. + # Must precede the form-element test below. + if SIGN_IN_MARKER in body: + return Outcome.AUTH_FAIL + + # The closed landing page, if a redirect was followed through to it. + if CLOSED_MARKER in body and LOAD_DATA not in body: + return Outcome.CLOSED + + # A re-rendered form means the submission did not take. Measured rejects come + # back 400, but keying on the form element as well as the status is stronger. + if status == 400 or FORM_ELEMENT in body: + return Outcome.REJECTED + + # Terminal confirmation page: no form element, positive recorded marker. + if _has_any(body, RECORDED_MARKERS): + return Outcome.RECORDED + + return Outcome.UNKNOWN diff --git a/tests/test_classify.py b/tests/test_classify.py new file mode 100644 index 0000000..06c3156 --- /dev/null +++ b/tests/test_classify.py @@ -0,0 +1,77 @@ +import unittest +from pathlib import Path + +from fastform.classify import Outcome, classify + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" + + +class TestClassifyRealFixtures(unittest.TestCase): + """Ground truth: bodies captured from real submissions to F1. + + resp-success.bin is the case an earlier classifier got wrong (reported + REJECTED on a recorded submission), so it is the most important row here. + """ + + def test_success_is_recorded(self): + body = (FIXTURES / "resp-success.bin").read_bytes() + self.assertEqual(classify(200, None, body), Outcome.RECORDED) + + def test_rejected_is_rejected(self): + body = (FIXTURES / "resp-rejected.bin").read_bytes() + self.assertEqual(classify(400, None, body), Outcome.REJECTED) + + def test_closed_landing_page_is_closed(self): + body = (FIXTURES / "resp-closed.bin").read_bytes() + self.assertEqual(classify(200, None, body), Outcome.CLOSED) + + +class TestClassifyRedirects(unittest.TestCase): + def test_302_to_closedform_is_closed(self): + self.assertEqual( + classify(302, "https://docs.google.com/forms/u/0/d/e/X/closedform", b""), + Outcome.CLOSED, + ) + + def test_302_to_accounts_is_auth_fail(self): + self.assertEqual( + classify(302, "https://accounts.google.com/ServiceLogin", b""), + Outcome.AUTH_FAIL, + ) + + def test_other_redirect_is_unknown(self): + self.assertEqual( + classify(302, "https://docs.google.com/forms/d/e/X/viewform", b""), + Outcome.UNKNOWN, + ) + + +class TestClassifyStructural(unittest.TestCase): + def test_signed_out_200_is_auth_fail_not_rejected(self): + # A signed-out sign-in-required form is served as 200 with the whole + # form. Without the sign-in check this reaches the form-element branch + # and reports REJECTED, sending the operator after a phantom payload bug. + body = ( + b'
' + b"" + b'' + ) + self.assertEqual(classify(200, None, body), Outcome.AUTH_FAIL) + + def test_form_element_on_200_is_rejected(self): + body = b'' + self.assertEqual(classify(200, None, body), Outcome.REJECTED) + + def test_terminal_page_without_markers_is_unknown(self): + # No form, but no recorded marker either: never guess RECORDED. + self.assertEqual(classify(200, None, b"something else"), Outcome.UNKNOWN) + + def test_server_error_is_unknown(self): + self.assertEqual(classify(500, None, b"oops"), Outcome.UNKNOWN) + + def test_empty_body_is_unknown(self): + self.assertEqual(classify(200, None, b""), Outcome.UNKNOWN) + + +if __name__ == "__main__": + unittest.main() From 40500a45015419454a67ffe19190b4105de80f65 Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:47:30 +0700 Subject: [PATCH 23/24] Add warm connection transport --- fastform/transport.py | 86 +++++++++++++++++++++++++++++++++++++++++ tests/test_transport.py | 80 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 fastform/transport.py create mode 100644 tests/test_transport.py diff --git a/fastform/transport.py b/fastform/transport.py new file mode 100644 index 0000000..32e7418 --- /dev/null +++ b/fastform/transport.py @@ -0,0 +1,86 @@ +"""Hold a connection open so sending costs a socket write, not a handshake. + +A cold DNS + TCP + TLS setup to docs.google.com on residential internet is +150-350ms and highly variable — bigger and jitterier than anything else in the +path. Connecting early and heartbeating turns that into zero. +""" + +import http.client +import time +from dataclasses import dataclass + +from fastform.compile import RequestBlob +from fastform.cookies import USER_AGENT + + +@dataclass +class Response: + status: int + location: str | None + body: bytes + elapsed_ms: float + + +class WarmConnection: + """One pre-established HTTP connection, reused across requests.""" + + def __init__( + self, + host: str, + port: int | None = None, + timeout: float = 15.0, + use_tls: bool = True, + ): + self.host = host + self.port = port + self.timeout = timeout + self.use_tls = use_tls + self._conn: http.client.HTTPConnection | None = None + + def connect(self) -> None: + cls = ( + http.client.HTTPSConnection if self.use_tls else http.client.HTTPConnection + ) + self._conn = cls(self.host, port=self.port, timeout=self.timeout) + self._conn.connect() + + def close(self) -> None: + if self._conn is not None: + self._conn.close() + self._conn = None + + def _request(self, method, path, body, headers) -> Response: + if self._conn is None: + raise RuntimeError("connect() must be called before sending") + started = time.perf_counter_ns() + self._conn.request(method, path, body=body, headers=headers) + resp = self._conn.getresponse() + payload = resp.read() # must drain fully or the connection cannot be reused + elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000 + return Response( + status=resp.status, + location=resp.getheader("Location"), + body=payload, + elapsed_ms=elapsed_ms, + ) + + def get(self, path: str, cookie: str) -> Response: + return self._request( + "GET", path, None, {"Cookie": cookie, "User-Agent": USER_AGENT} + ) + + def send(self, blob: RequestBlob) -> Response: + return self._request("POST", blob.path, blob.body, blob.headers) + + def heartbeat(self, path: str, cookie: str) -> bool: + """Keep the socket and any NAT mapping alive. Reconnects on failure.""" + try: + self.get(path, cookie) + return True + except (OSError, http.client.HTTPException): + self.close() + try: + self.connect() + return True + except OSError: + return False diff --git a/tests/test_transport.py b/tests/test_transport.py new file mode 100644 index 0000000..473be7e --- /dev/null +++ b/tests/test_transport.py @@ -0,0 +1,80 @@ +import http.server +import threading +import unittest + +from fastform.compile import RequestBlob +from fastform.transport import Response, WarmConnection + + +class _Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b"echo:" + body) + + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, *args): + pass + + +class TestWarmConnection(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.server = http.server.HTTPServer(("127.0.0.1", 0), _Handler) + cls.port = cls.server.server_address[1] + cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) + cls.thread.start() + + @classmethod + def tearDownClass(cls): + cls.server.shutdown() + cls.server.server_close() + + def _conn(self): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False) + conn.connect() + self.addCleanup(conn.close) + return conn + + def test_send_returns_body_and_status(self): + conn = self._conn() + blob = RequestBlob( + host="127.0.0.1", path="/x", body="a=1", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + resp = conn.send(blob) + self.assertIsInstance(resp, Response) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.body, b"echo:a=1") + + def test_records_elapsed_time(self): + conn = self._conn() + blob = RequestBlob(host="127.0.0.1", path="/x", body="a=1", headers={}) + self.assertGreater(conn.send(blob).elapsed_ms, 0.0) + + def test_connection_is_reusable(self): + conn = self._conn() + blob = RequestBlob(host="127.0.0.1", path="/x", body="a=1", headers={}) + self.assertEqual(conn.send(blob).status, 200) + self.assertEqual(conn.send(blob).status, 200) + + def test_heartbeat_keeps_it_alive(self): + conn = self._conn() + self.assertTrue(conn.heartbeat("/ping", "SID=x")) + + def test_send_before_connect_raises(self): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False) + blob = RequestBlob(host="127.0.0.1", path="/x", body="", headers={}) + with self.assertRaises(RuntimeError): + conn.send(blob) + + +if __name__ == "__main__": + unittest.main() From 5fb0022cd5e1ec1bcd29b82e90152cc1262aecc5 Mon Sep 17 00:00:00 2001 From: ohm Date: Thu, 6 Aug 2026 09:55:42 +0700 Subject: [PATCH 24/24] Make the connection-reuse test a real guard; fix double-connect leak Review found test_connection_is_reusable could not fail: the test server defaulted to HTTP/1.0 and closed after every response, so http.client silently opened a fresh socket and the test passed even with the body-drain deleted - guarding nothing, on the one property this module exists to provide. Fix: - Test handler now speaks HTTP/1.1 with Content-Length on every response, so the server actually keeps the connection alive. - test_connection_is_reusable asserts the same underlying socket serves both requests, with a 5s timeout so a drain regression fails fast, not by hanging. Verified: deleting resp.read() makes it fail (ResponseNotReady); restoring it passes. - connect() now closes any existing connection before reassigning, fixing a socket leak on a second connect(). Guarded by a new test asserting the first socket's fileno() is -1 after reconnect. - Added the missing get()-before-connect RuntimeError test. 93 tests pass. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HvAS3iP5DJxYkmGuwXadoL --- fastform/transport.py | 2 ++ tests/test_transport.py | 47 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/fastform/transport.py b/fastform/transport.py index 32e7418..fd61978 100644 --- a/fastform/transport.py +++ b/fastform/transport.py @@ -38,6 +38,8 @@ def __init__( self._conn: http.client.HTTPConnection | None = None def connect(self) -> None: + if self._conn is not None: + self._conn.close() cls = ( http.client.HTTPSConnection if self.use_tls else http.client.HTTPConnection ) diff --git a/tests/test_transport.py b/tests/test_transport.py index 473be7e..11647ea 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -7,18 +7,29 @@ class _Handler(http.server.BaseHTTPRequestHandler): + # HTTP/1.1 so the server keeps the connection alive between requests. Under + # the default HTTP/1.0 the server closes after every response, http.client + # silently opens a fresh socket, and test_connection_is_reusable would pass + # even if WarmConnection failed to drain and reuse — i.e. it could not fail. + # HTTP/1.1 keep-alive requires a correct Content-Length on every response. + protocol_version = "HTTP/1.1" + def do_POST(self): length = int(self.headers.get("Content-Length", 0)) body = self.rfile.read(length) + payload = b"echo:" + body self.send_response(200) self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(payload))) self.end_headers() - self.wfile.write(b"echo:" + body) + self.wfile.write(payload) def do_GET(self): + payload = b"ok" self.send_response(200) + self.send_header("Content-Length", str(len(payload))) self.end_headers() - self.wfile.write(b"ok") + self.wfile.write(payload) def log_message(self, *args): pass @@ -37,8 +48,8 @@ def tearDownClass(cls): cls.server.shutdown() cls.server.server_close() - def _conn(self): - conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False) + def _conn(self, timeout=15.0): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False, timeout=timeout) conn.connect() self.addCleanup(conn.close) return conn @@ -60,10 +71,17 @@ def test_records_elapsed_time(self): self.assertGreater(conn.send(blob).elapsed_ms, 0.0) def test_connection_is_reusable(self): - conn = self._conn() + # A real reuse guard: the SAME underlying socket must serve both + # requests. If WarmConnection stopped draining the body, http.client + # could not reuse the keep-alive socket; the short timeout makes that + # regression fail fast here instead of hanging the suite. + conn = self._conn(timeout=5.0) blob = RequestBlob(host="127.0.0.1", path="/x", body="a=1", headers={}) self.assertEqual(conn.send(blob).status, 200) + first_sock = id(conn._conn.sock) self.assertEqual(conn.send(blob).status, 200) + self.assertEqual(id(conn._conn.sock), first_sock, + "second request opened a new socket — connection not reused") def test_heartbeat_keeps_it_alive(self): conn = self._conn() @@ -75,6 +93,25 @@ def test_send_before_connect_raises(self): with self.assertRaises(RuntimeError): conn.send(blob) + def test_get_before_connect_raises(self): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False) + with self.assertRaises(RuntimeError): + conn.get("/x", "SID=x") + + def test_connect_twice_closes_the_first_socket(self): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False) + conn.connect() + self.addCleanup(conn.close) + conn.send(RequestBlob(host="127.0.0.1", path="/x", body="a=1", headers={})) + first = conn._conn.sock + conn.connect() # must close `first` rather than leak it + self.assertEqual(first.fileno(), -1, "first socket was leaked on reconnect") + # the fresh connection still works + self.assertEqual( + conn.send(RequestBlob(host="127.0.0.1", path="/x", body="b=2", headers={})).status, + 200, + ) + if __name__ == "__main__": unittest.main()