diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4f4b14d --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +config/cookies.txt +__pycache__/ +*.pyc +.DS_Store +fixtures/*.curl.txt diff --git a/config/answers.toml b/config/answers.toml new file mode 100644 index 0000000..488f332 --- /dev/null +++ b/config/answers.toml @@ -0,0 +1,17 @@ +# Answer bank. Each entry matches a question by text and supplies a value. +# match — list of phrases; scored against question text (see fastform/match.py) +# regex — optional; a hit here is an exact match, outranking everything +# value — what gets submitted + +[[answer]] +match = ["full name", "your name", "name"] +value = "Test Person" + +[[answer]] +match = ["email address", "email"] +regex = "e-?mail" +value = "test@example.com" + +[[answer]] +match = ["phone number", "phone", "mobile"] +value = "0800000000" diff --git a/config/profile.toml b/config/profile.toml new file mode 100644 index 0000000..a3f2b7a --- /dev/null +++ b/config/profile.toml @@ -0,0 +1,22 @@ +# Runtime configuration. + +[chrome] +# Which Chrome profile's cookies to submit with. REQUIRED before any real run — +# the tool refuses to guess, because picking the wrong profile submits as the +# wrong person. Leave unset until you know which Google account competes. +# +# To see the choices, run: +# python3 -c "from fastform import cookies; [print(f'{p.name}: {e}') for p,e in cookies.resolve_accounts()]" +# +# That prints each profile with the account it is signed into. Then set: +# profile = "Profile 1" +# +# Identified on this machine so far: +# Profile 1 -> student@example.com +# Profile 5 -> personal@example.com +# +# NEITHER is the competition account? Sign it into any Chrome profile first, +# then re-run the command above — a freshly signed-in profile shows up there +# and can be pinned like any other. + +# profile = diff --git a/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md b/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md new file mode 100644 index 0000000..165b24f --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md @@ -0,0 +1,2975 @@ +# FastForm Phases 0–4 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a hand-run tool that takes a Google Form URL, parses its structure, fills it from a prepared answer bank, submits it in a single authenticated POST, and reports what happened. + +**Architecture:** Pure-function pipeline with a thin CLI over it — `parse` (HTML → `FormSpec`), `match` (`FormSpec` + answer bank → `FilledForm`), `compile` (`FilledForm` → `RequestBlob`), `transport` (warm connection, send), `classify` (response bytes → `Outcome`). Every stage is independently testable with no network. The CLI wires them together and is the only place that touches both the network and the domain types. + +**Tech Stack:** Python 3.14 (3.11+ required for `tomllib`), stdlib only. Tests via `unittest`. No pytest, no requests, no Playwright. + +**Source spec:** `docs/superpowers/specs/2026-08-05-fastform-design.md` + +## Global Constraints + +- **Stdlib only.** No third-party packages in `fastform/`. If a task seems to need one, stop and raise it. +- **Python 3.11+** (`tomllib`). Development target is 3.14.6. +- **Test runner:** `python3 -m unittest discover -s tests -v` from the repo root. +- **No network in unit tests.** Network verification is done by explicit manual commands with stated expected output. +- **`config/cookies.txt` is never committed.** It goes in `.gitignore` in Task 1. +- **Repo root:** `/Users/supa/projects/active/fastform`. All paths below are relative to it. +- **Scope:** This plan covers spec Phases 0–4 only. Phase 5 (calibration) and beyond require measurements that do not exist yet and get their own plan. + +## Scope Note + +Two deliberate deviations from the spec, both to avoid speculative work: + +1. **Cookies are captured manually** from Chrome DevTools into `config/cookies.txt`, not decrypted from Chrome's SQLite store. Decryption needs AES, which needs a dependency or an `openssl` subprocess, and Google session cookies last months. Automating this is deferred until it proves annoying. +2. **`RequestBlob` holds an encoded body string plus a headers dict**, not raw socket bytes. The costly part (URL-encoding the payload) still happens at arm time. Header serialization by `http.client` is microseconds. Spec experiment E7 decides whether raw-byte serialization is worth building. + +--- + +### Task 1: Scaffold the repo and create test form F1 + +**Files:** +- Create: `.gitignore` +- Create: `fastform/__init__.py` +- Create: `tests/__init__.py` +- Create: `config/answers.toml` +- Create: `config/cookies.txt` (untracked) +- Create: `fixtures/README.md` + +**Interfaces:** +- Consumes: nothing +- Produces: the package layout every later task imports from, and the F1 test form that Task 3's gate test targets. + +- [ ] **Step 1: Create the directory layout** + +```bash +cd /Users/supa/projects/active/fastform +mkdir -p fastform tests config fixtures +touch fastform/__init__.py tests/__init__.py +``` + +- [ ] **Step 2: Write `.gitignore`** + +Create `.gitignore`: + +``` +config/cookies.txt +__pycache__/ +*.pyc +.DS_Store +``` + +- [ ] **Step 3: Create test form F1 in Google Forms** + +This is manual work in a browser. Go to https://forms.google.com and create a form with these exact properties: + +- Title: `FastForm F1` +- Settings → Responses → **Collect email addresses: Verified** +- Settings → Responses → **Limit to 1 response: OFF** +- One section only. +- Three questions, all **Short answer**, all **Required**: + 1. `What is your full name?` + 2. `What is your email address?` + 3. `What is your phone number?` + +**Why these two settings matter.** "Verified" is what forces sign-in, so the +authenticated POST path actually gets exercised — and unlike "Responder input" +it does not add a question we would have to answer, because Google fills the +address from the session. Limit-to-one stays **off** so F1 remains submittable +for the whole plan; Tasks 3, 8, 9, and 11 all submit to it repeatedly. Task 9 +turns limit-to-one on temporarily, at the one point where it is needed. + +Click **Send** → link icon → copy the URL. Save it somewhere you can paste later; it looks like `https://docs.google.com/forms/d/e//viewform`. + +- [ ] **Step 3a: Confirm the form really does require sign-in** + +Open the F1 URL in a **private/incognito window**. + +Expected: Google asks you to sign in before showing the form. If the form +renders without asking, "Verified" did not take effect — go back to +Settings → Responses and set it. Everything downstream assumes an authenticated +form, and a form that does not need auth would let the whole plan pass while +proving nothing. + +- [ ] **Step 4: Submit F1 once by hand and capture the request** + +Open the F1 URL in Chrome, signed into the Google account you will compete with. + +1. Open DevTools (Cmd+Opt+I) → **Network** tab. +2. Check **Preserve log**. +3. Fill in the three answers and click Submit. +4. In the Network list, find the request named `formResponse` (method POST). +5. Right-click it → **Copy** → **Copy as cURL**. +6. Paste into `fixtures/f1-manual-submit.curl.txt`. + +This file is the ground truth Task 8 diffs against. It contains your cookies, so add it to `.gitignore`: + +```bash +echo "fixtures/*.curl.txt" >> .gitignore +``` + +- [ ] **Step 5: Capture your cookie header** + +In the same DevTools Network panel, click the `formResponse` request → **Headers** → **Request Headers** → find `Cookie:`. Copy the entire value (it is long). + +Write it to `config/cookies.txt` as a single line, **value only — do not include the leading `Cookie:`**. The Python loader in Task 2 strips that prefix as a safety net, but the shell `curl` in the next step does not, and a doubled `Cookie: Cookie:` header silently produces a signed-out page. + +- [ ] **Step 6: Save the form page HTML as a fixture** + +```bash +curl -s -H "Cookie: $(cat config/cookies.txt)" \ + '' > fixtures/f1-viewform.html +wc -c fixtures/f1-viewform.html +``` + +Expected: a byte count in the tens of thousands (typically 50,000–200,000). If it is under 5,000 you got a sign-in redirect page instead — recheck the cookie. + +Verify the form data is present: + +```bash +grep -c FB_PUBLIC_LOAD_DATA_ fixtures/f1-viewform.html +``` + +Expected: `1` + +- [ ] **Step 7: Write `fixtures/README.md`** + +```markdown +# Fixtures + +Captured artifacts used as test data. Nothing here is generated by code. + +- `f1-viewform.html` — F1 form page, signed in. Input for parse tests. +- `f1-manual-submit.curl.txt` — DevTools "copy as cURL" of a real manual + submission to F1. Ground truth for the Task 8 request diff. Untracked: + contains cookies. + +Response fixtures (`resp-*.bin`) are captured in Task 9. +``` + +- [ ] **Step 8: Write a starter `config/answers.toml`** + +Use values matching what you typed into F1 in Step 4, so later tasks can assert against them. + +```toml +# Answer bank. Each entry matches a question by text and supplies a value. +# match — list of phrases; scored against question text (see fastform/match.py) +# regex — optional; a hit here is an exact match, outranking everything +# value — what gets submitted + +[[answer]] +match = ["full name", "your name", "name"] +value = "Test Person" + +[[answer]] +match = ["email address", "email"] +regex = "e-?mail" +value = "test@example.com" + +[[answer]] +match = ["phone number", "phone", "mobile"] +value = "0800000000" +``` + +- [ ] **Step 9: Commit** + +```bash +git add .gitignore fastform/ tests/ config/answers.toml fixtures/README.md +git commit -m "Scaffold package layout and answer bank" +``` + +Confirm `config/cookies.txt` and `fixtures/*.curl.txt` are NOT in the commit: + +```bash +git show --stat --name-only HEAD +``` + +Expected: no `cookies.txt`, no `.curl.txt`. + +--- + +### Task 2: Cookie loading and freshness check + +**Files:** +- Create: `fastform/cookies.py` +- Create: `tests/test_cookies.py` + +**Interfaces:** +- Consumes: `config/cookies.txt` from Task 1. +- Produces: + - `load(path: Path | None = None) -> str` — returns a bare cookie header value. + - `verify(cookie_header: str) -> tuple[bool, str | None]` — `(signed_in, email_or_none)`. Network call. + - `USER_AGENT: str` — the UA string every later task sends. + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_cookies.py`: + +```python +import tempfile +import unittest +from pathlib import Path + +from fastform import cookies + + +class TestLoad(unittest.TestCase): + def _write(self, text): + tmp = Path(tempfile.mkdtemp()) / "cookies.txt" + tmp.write_text(text, encoding="utf-8") + return tmp + + def test_returns_bare_value(self): + path = self._write("SID=abc; HSID=def\n") + self.assertEqual(cookies.load(path), "SID=abc; HSID=def") + + def test_strips_cookie_prefix(self): + path = self._write("Cookie: SID=abc; HSID=def") + self.assertEqual(cookies.load(path), "SID=abc; HSID=def") + + def test_strips_lowercase_prefix(self): + path = self._write("cookie: SID=abc") + self.assertEqual(cookies.load(path), "SID=abc") + + def test_empty_file_raises(self): + path = self._write(" \n") + with self.assertRaises(ValueError): + cookies.load(path) + + def test_missing_file_raises(self): + path = Path(tempfile.mkdtemp()) / "nope.txt" + with self.assertRaises(FileNotFoundError): + cookies.load(path) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +cd /Users/supa/projects/active/fastform +python3 -m unittest tests.test_cookies -v +``` + +Expected: `ModuleNotFoundError: No module named 'fastform.cookies'` + +- [ ] **Step 3: Write the implementation** + +Create `fastform/cookies.py`: + +```python +"""Load a Cookie header captured from Chrome DevTools and check it still authenticates. + +Cookies are captured by hand rather than decrypted out of Chrome's SQLite +store: decryption needs AES (a dependency), and Google session cookies last +months. See the plan's Scope Note. +""" + +import http.client +import re +from pathlib import Path + +USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36" +) + +DEFAULT_PATH = Path(__file__).resolve().parent.parent / "config" / "cookies.txt" + +_EMAIL_RE = re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+") + + +def load(path: Path | None = None) -> str: + """Return the bare Cookie header value, with any 'Cookie:' prefix removed.""" + path = path or DEFAULT_PATH + raw = path.read_text(encoding="utf-8").strip() + if raw.lower().startswith("cookie:"): + raw = raw.split(":", 1)[1].strip() + if not raw: + raise ValueError(f"{path} is empty — paste a Cookie header into it") + return raw + + +def verify(cookie_header: str) -> tuple[bool, str | None]: + """GET a signed-in-only page. Returns (signed_in, email_if_found). + + Signed-out sessions get redirected to accounts.google.com, which is a + structural signal and does not depend on the account's UI language. + """ + conn = http.client.HTTPSConnection("docs.google.com", timeout=15) + try: + conn.request( + "GET", + "/forms/u/0/", + headers={"Cookie": cookie_header, "User-Agent": USER_AGENT}, + ) + resp = conn.getresponse() + location = resp.getheader("Location") or "" + body = resp.read() + finally: + conn.close() + + if resp.status in (301, 302, 303, 307, 308) and "accounts.google.com" in location: + return False, None + if resp.status != 200: + return False, None + + match = _EMAIL_RE.search(body.decode("utf-8", "replace")) + return True, match.group(0) if match else None +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +python3 -m unittest tests.test_cookies -v +``` + +Expected: `Ran 5 tests` … `OK` + +- [ ] **Step 5: Manually verify the cookie actually works** + +```bash +python3 -c " +from fastform import cookies +c = cookies.load() +print(cookies.verify(c)) +" +``` + +Expected: `(True, 'your.email@gmail.com')` — or `(True, None)` if no email appears in the page, which is still a pass. + +If you get `(False, None)`: the cookie is stale or was copied incompletely. Redo Task 1 Step 5. **Do not continue until this returns `True`.** + +- [ ] **Step 6: Commit** + +```bash +git add fastform/cookies.py tests/test_cookies.py +git commit -m "Add cookie loading and freshness check" +``` + +--- + +### Task 2b: Read cookies live from Chrome's cookie store + +**Files:** +- Modify: `fastform/cookies.py` +- Modify: `tests/test_cookies.py` + +**Interfaces:** +- Consumes: nothing new. +- Produces: + - `ChromeProfile` dataclass: `name: str`, `path: Path`, `google_cookies: int` + - `list_profiles() -> list[ChromeProfile]` + - `keychain_secret() -> str` + - `derive_key(secret: str) -> bytes` + - `decrypt_value(blob: bytes, key: bytes) -> str | None` + - `cookie_header_from_chrome(profile: str | None = None) -> str` + - `ChromeCookieError` exception + - existing `load()`, `verify()`, `USER_AGENT` stay as they are + +**Why this task exists.** Task 2 reads a Cookie header captured by hand. Measurement showed such a header authenticates for roughly seven minutes and is dead across all of Google within twenty: Chrome continuously rotates `__Secure-1PSIDTS`, and once it does, the captured value stops working. A static header cannot support a boot-time check hours early, cannot support `ARMED_HOLDING`, and barely supports a hand-run tool. `load()` survives only as a manual override. + +**Every mechanism below was verified end to end on this machine** — 91/91 Google cookies decrypted, and the resulting header authenticated against a real form. Zero Python dependencies. + +- Keychain secret: `security find-generic-password -w -s "Chrome Safe Storage"` +- Key: `PBKDF2-HMAC-SHA1(secret, b"saltysalt", 1003, dklen=16)` — `hashlib`, stdlib +- Cipher: AES-128-CBC, IV of sixteen spaces, values prefixed `v10`/`v11`. No stdlib AES, so shell out to `openssl enc -d -aes-128-cbc -K -iv -nopad` +- Strip PKCS7 padding, then strip a 32-byte domain-hash prefix if present (newer Chrome adds one; detect by the first 32 bytes not being printable ASCII) +- Chrome holds a lock on the live DB — copy it before opening +- Filter `host_key IN ('.google.com', 'docs.google.com', '.docs.google.com')`; this machine had 91 Google cookies but only 24 relevant ones, and each decrypt is a subprocess fork, so filtering first roughly quarters the cost + +**Profiles matter.** This machine has two signed-in Chrome profiles on different Google accounts (`Profile 1` and `Profile 5`). Picking the wrong one submits as the wrong person. `list_profiles()` exists so the operator can see the choice. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_cookies.py`: + +```python +import sqlite3 +import subprocess +import tempfile +import unittest +from pathlib import Path + +from fastform import cookies + + +class TestDeriveKey(unittest.TestCase): + def test_key_is_16_bytes(self): + self.assertEqual(len(cookies.derive_key("abc")), 16) + + def test_deterministic(self): + self.assertEqual(cookies.derive_key("abc"), cookies.derive_key("abc")) + + def test_differs_per_secret(self): + self.assertNotEqual(cookies.derive_key("abc"), cookies.derive_key("abd")) + + +class TestDecryptValue(unittest.TestCase): + """Round-trip against openssl so the test proves the real cipher path.""" + + KEY = cookies.derive_key("test-secret") + + def _encrypt(self, plaintext: bytes) -> bytes: + proc = subprocess.run( + ["openssl", "enc", "-aes-128-cbc", "-K", self.KEY.hex(), + "-iv", (b" " * 16).hex()], + input=plaintext, capture_output=True, check=True, + ) + return b"v10" + proc.stdout + + def test_round_trip(self): + blob = self._encrypt(b"SID=hello-world") + self.assertEqual(cookies.decrypt_value(blob, self.KEY), "SID=hello-world") + + def test_unencrypted_blob_returns_none(self): + self.assertIsNone(cookies.decrypt_value(b"plaintext", self.KEY)) + + def test_empty_blob_returns_none(self): + self.assertIsNone(cookies.decrypt_value(b"", self.KEY)) + + def test_wrong_key_does_not_raise(self): + blob = self._encrypt(b"SID=hello-world") + other = cookies.derive_key("different") + result = cookies.decrypt_value(blob, other) + self.assertNotEqual(result, "SID=hello-world") + + +class TestListProfiles(unittest.TestCase): + def _profile_dir(self, name, rows): + root = Path(tempfile.mkdtemp()) + prof = root / name + prof.mkdir() + con = sqlite3.connect(prof / "Cookies") + con.execute( + "CREATE TABLE cookies (host_key TEXT, name TEXT, encrypted_value BLOB)" + ) + con.executemany("INSERT INTO cookies VALUES (?,?,?)", rows) + con.commit() + con.close() + return root + + def test_counts_google_cookies(self): + root = self._profile_dir( + "Profile 1", + [(".google.com", "SID", b"v10x"), ("example.com", "other", b"v10y")], + ) + profiles = cookies.list_profiles(root) + self.assertEqual(len(profiles), 1) + self.assertEqual(profiles[0].name, "Profile 1") + self.assertEqual(profiles[0].google_cookies, 1) + + def test_skips_profiles_without_a_cookie_db(self): + root = Path(tempfile.mkdtemp()) + (root / "Profile 9").mkdir() + self.assertEqual(cookies.list_profiles(root), []) + + def test_missing_root_returns_empty(self): + self.assertEqual( + cookies.list_profiles(Path(tempfile.mkdtemp()) / "nope"), [] + ) +``` + +- [ ] **Step 2: Run to verify they fail** + +```bash +cd /Users/supa/projects/active/fastform +python3 -m unittest tests.test_cookies -v +``` + +Expected: `AttributeError: module 'fastform.cookies' has no attribute 'derive_key'` + +- [ ] **Step 3: Write the implementation** + +Append to `fastform/cookies.py`: + +```python +import hashlib +import shutil +import sqlite3 +import subprocess +import tempfile +from dataclasses import dataclass + +#: Chrome's macOS cookie encryption, all constants fixed by Chromium itself. +_SALT = b"saltysalt" +_ITERATIONS = 1003 +_KEY_LENGTH = 16 +_IV = b" " * 16 +_ENCRYPTED_PREFIXES = (b"v10", b"v11") + +#: Only these host_keys are sent to docs.google.com. Filtering here matters: +#: each value costs one openssl fork, and this cuts ~91 cookies down to ~24. +_GOOGLE_HOSTS = (".google.com", "docs.google.com", ".docs.google.com") + +CHROME_ROOT = Path.home() / "Library/Application Support/Google/Chrome" + + +class ChromeCookieError(Exception): + """Chrome's cookie store could not be read or decrypted.""" + + +@dataclass +class ChromeProfile: + name: str + path: Path + google_cookies: int + + +def keychain_secret() -> str: + """Chrome's encryption password, stored in the login Keychain. + + macOS may prompt for permission the first time. That prompt is the reason + this is worth doing well before the event rather than during it. + """ + proc = subprocess.run( + ["security", "find-generic-password", "-w", "-s", "Chrome Safe Storage"], + capture_output=True, text=True, timeout=30, + ) + if proc.returncode != 0: + raise ChromeCookieError( + f"could not read Chrome Safe Storage from Keychain: {proc.stderr.strip()}" + ) + return proc.stdout.strip() + + +def derive_key(secret: str) -> bytes: + return hashlib.pbkdf2_hmac( + "sha1", secret.encode(), _SALT, _ITERATIONS, _KEY_LENGTH + ) + + +def decrypt_value(blob: bytes, key: bytes) -> str | None: + """Decrypt one cookie value. Returns None for anything not v10/v11. + + Python has no stdlib AES, so this shells out to openssl rather than taking + a dependency. -nopad is required because we strip PKCS7 ourselves; letting + openssl do it makes a wrong key raise instead of returning garbage. + """ + if not blob or blob[:3] not in _ENCRYPTED_PREFIXES: + return None + proc = subprocess.run( + ["openssl", "enc", "-d", "-aes-128-cbc", + "-K", key.hex(), "-iv", _IV.hex(), "-nopad"], + input=blob[3:], capture_output=True, + ) + if proc.returncode != 0 or not proc.stdout: + return None + out = proc.stdout + pad = out[-1] + if 1 <= pad <= 16: + out = out[:-pad] + # Chrome >= v24 prefixes 32 bytes of domain hash before the value. + if len(out) > 32 and not re.fullmatch(rb"[\x20-\x7e]*", out[:32]): + out = out[32:] + return out.decode("utf-8", "replace") + + +def _read_cookies(db_path: Path) -> list[tuple[str, bytes]]: + """Copy first — Chrome holds a lock on the live database.""" + with tempfile.TemporaryDirectory() as tmp: + copy = Path(tmp) / "Cookies" + shutil.copy(db_path, copy) + con = sqlite3.connect(copy) + try: + placeholders = ",".join("?" * len(_GOOGLE_HOSTS)) + return con.execute( + f"SELECT name, encrypted_value FROM cookies " + f"WHERE host_key IN ({placeholders})", _GOOGLE_HOSTS + ).fetchall() + finally: + con.close() + + +def list_profiles(root: Path | None = None) -> list[ChromeProfile]: + """Every Chrome profile holding Google cookies, most cookies first. + + Profiles matter: a machine can have several signed into different accounts, + and the wrong one submits as the wrong person. + """ + root = root or CHROME_ROOT + if not root.is_dir(): + return [] + found = [] + for child in sorted(root.iterdir()): + db = child / "Cookies" + if not db.is_file(): + continue + try: + count = len(_read_cookies(db)) + except (sqlite3.Error, OSError): + continue + if count: + found.append(ChromeProfile(child.name, child, count)) + return sorted(found, key=lambda p: -p.google_cookies) + + +def cookie_header_from_chrome(profile: str | None = None) -> str: + """Build a Cookie header from Chrome's live store. + + Read at call time, never cached: Chrome rotates __Secure-1PSIDTS + continuously, and a header captured twenty minutes ago no longer + authenticates anywhere on Google. + """ + profiles = list_profiles() + if not profiles: + raise ChromeCookieError(f"no Chrome profile with Google cookies under {CHROME_ROOT}") + if profile: + chosen = next((p for p in profiles if p.name == profile), None) + if chosen is None: + names = ", ".join(p.name for p in profiles) + raise ChromeCookieError(f"no profile named {profile!r}; found: {names}") + else: + chosen = profiles[0] + + key = derive_key(keychain_secret()) + jar: dict[str, str] = {} + for name, blob in _read_cookies(chosen.path / "Cookies"): + value = decrypt_value(blob, key) + if value is not None: + jar.setdefault(name, value) + if not jar: + raise ChromeCookieError(f"decrypted no cookies from profile {chosen.name!r}") + return "; ".join(f"{k}={v}" for k, v in jar.items()) +``` + +- [ ] **Step 4: Run the tests** + +```bash +python3 -m unittest tests.test_cookies -v +``` + +Expected: all tests pass, old and new. + +- [ ] **Step 5: Verify against the real Chrome store** + +```bash +python3 -c " +from fastform import cookies +for p in cookies.list_profiles(): + print(f'{p.name:12} {p.google_cookies} google cookies') +h = cookies.cookie_header_from_chrome() +print(f'header: {len(h)} chars, {h.count(\"=\")} cookies') +print('authenticates:', cookies.verify(h)) +" +``` + +Expected: at least one profile listed, a header of a couple of thousand characters, and `verify` returning `(True, ...)`. + +macOS may show a Keychain prompt on first run — approve it. That prompt is exactly why this must be exercised well before the event. + +- [ ] **Step 6: Commit** + +```bash +git add fastform/cookies.py tests/test_cookies.py +git commit -m "Read cookies live from Chrome's store" +``` + +--- + +### Task 3: E1 gate — prove a cookie-only POST records a response + +**Files:** +- Create: `tools/e1_gate.py` + +**Interfaces:** +- Consumes: `fastform.cookies.load`, `fastform.cookies.USER_AGENT` from Task 2. +- Produces: a yes/no answer to whether the entire design is viable. No code downstream depends on this file. + +This task is a spike, not a component. It is deliberately hardcoded and throwaway. **If it fails, stop the project and re-plan** — every later task assumes it passed. + +- [ ] **Step 1: Read the entry IDs out of your captured cURL** + +```bash +grep -o 'entry\.[0-9]*' fixtures/f1-manual-submit.curl.txt | sort -u +``` + +Expected: three lines, e.g. `entry.1234567890`, `entry.987654321`, `entry.55555555`. + +Also get the form action path: + +```bash +grep -o '/forms/d/e/[^/]*/formResponse' fixtures/f1-manual-submit.curl.txt | head -1 +``` + +Expected: one path like `/forms/d/e/1FAIpQLS.../formResponse` + +- [ ] **Step 2: Write the spike** + +Create `tools/e1_gate.py`, substituting the four values you just found: + +```python +"""E1: does a cookie-only POST record a response? Spike — hardcoded on purpose. + +Run once. If this does not record a row in F1's response sheet, the whole +direct-POST design is wrong and the plan must be revisited. +""" + +import http.client +import sys +import urllib.parse +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from fastform import cookies # noqa: E402 + +ACTION_PATH = "/forms/d/e/REPLACE_WITH_YOUR_FORM_ID/formResponse" +FIELDS = [ + ("entry.REPLACE_NAME_ID", "E1 Gate Test"), + ("entry.REPLACE_EMAIL_ID", "e1@example.com"), + ("entry.REPLACE_PHONE_ID", "0800000000"), + ("fvv", "1"), + ("pageHistory", "0"), + ("fbzx", "-1234567890123456789"), + ("submissionTimestamp", "-1"), +] + + +def main() -> int: + cookie = cookies.load() + body = urllib.parse.urlencode(FIELDS, encoding="utf-8") + + conn = http.client.HTTPSConnection("docs.google.com", timeout=15) + try: + conn.request( + "POST", + ACTION_PATH, + body=body, + headers={ + "Cookie": cookie, + "User-Agent": cookies.USER_AGENT, + "Content-Type": "application/x-www-form-urlencoded", + "Referer": "https://docs.google.com" + ACTION_PATH.replace( + "/formResponse", "/viewform" + ), + }, + ) + resp = conn.getresponse() + payload = resp.read() + finally: + conn.close() + + print(f"status={resp.status}") + print(f"location={resp.getheader('Location')}") + print(f"bytes={len(payload)}") + Path("fixtures/resp-e1.bin").write_bytes(payload) + print("body written to fixtures/resp-e1.bin") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **Step 3: Run it** + +```bash +cd /Users/supa/projects/active/fastform +python3 tools/e1_gate.py +``` + +Expected: `status=200` and a non-trivial byte count. A `status=302` with a `location` on `accounts.google.com` means the cookie is not being accepted. + +- [ ] **Step 4: Check the response sheet — this is the actual gate** + +Open F1 in Google Forms → **Responses** tab. + +F1 has limit-to-one **off**, so this is a second row alongside the manual +submission from Task 1 — that is expected, and it is why limit-to-one is off. + +**PASS:** a new row appeared containing `E1 Gate Test`. The design is viable. Continue. + +**FAIL:** no new row. Stop here. Capture what you can and re-plan: + +```bash +head -c 2000 fixtures/resp-e1.bin +``` + +Compare the request in `tools/e1_gate.py` against `fixtures/f1-manual-submit.curl.txt` field by field — the cURL contains something the spike is missing. Common culprits are additional `entry.*_sentinel` fields, a `draftResponse` parameter, or a required header. Do not proceed to Task 4 until a row appears. + +- [ ] **Step 5: Commit** + +```bash +git add tools/e1_gate.py +git commit -m "Add E1 gate spike: cookie-only POST records a response" +``` + +--- + +### Task 4: Create test forms F3 and F4 + +**Files:** +- Modify: `fixtures/README.md` +- Create: `fixtures/f3-viewform.html` +- Create: `fixtures/f4-viewform.html` + +**Interfaces:** +- Consumes: nothing in code. +- Produces: the multi-section and validation fixtures Tasks 5 and 9 test against. + +Deferred until now on purpose: if Task 3 had failed, these forms would have been wasted effort. + +The spec's test bed pairs F1 (limit-to-one ON) with F2 (limit-to-one OFF). +This plan inverts that: F1 runs with limit-to-one **off** so it stays +submittable throughout, and Task 9 flips it on for one capture. That makes a +separate always-off F2 redundant here. The permanently-ON variant the spec calls +F1 is what Phase 5 will need, for E3's comparison of overwrite behavior with the +setting on versus off — it gets created in that plan. + +**Both forms below use Settings → Responses → Collect email addresses: Verified**, for the same reason F1 does — it forces sign-in without adding a question. + +- [ ] **Step 1: Create F3 — multi-section, mixed types** + +Title `FastForm F3`. Collect email addresses: **Verified**. Limit to 1 response: **OFF**. Three sections: + +**Section 1:** +- `What is your full name?` — Short answer, Required +- `Tell us about yourself` — Paragraph, Optional + +**Section 2:** +- `Which city?` — Multiple choice, Required. Options: `Bangkok`, `Chiang Mai`, `Phuket`, plus **Other** +- `Pick your sizes` — Checkboxes, Optional. Options: `Small`, `Medium`, `Large` +- `Choose a department` — Dropdown, Required. Options: `Engineering`, `Design`, `Sales` + +**Section 3:** +- `Rate your confidence` — Linear scale 1–5, Required +- `Preferred date` — Date, Optional +- `Preferred time` — Time, Optional + +Save its viewform URL. + +- [ ] **Step 2: Create F4 — validation and a file upload** + +Title `FastForm F4`. Collect email addresses: **Verified**. One section: + +- `Numeric code` — Short answer, Required, Response validation → Number → Between → `1000` and `9999` +- `Contact email` — Short answer, Required, Response validation → Text → Email address +- `Upload your entry` — **File upload**, Optional + +Save its viewform URL. + +- [ ] **Step 3: Capture F3 and F4 page fixtures** + +```bash +cd /Users/supa/projects/active/fastform +curl -s -H "Cookie: $(cat config/cookies.txt)" '' > fixtures/f3-viewform.html +curl -s -H "Cookie: $(cat config/cookies.txt)" '' > fixtures/f4-viewform.html +grep -c FB_PUBLIC_LOAD_DATA_ fixtures/f3-viewform.html fixtures/f4-viewform.html +``` + +Expected: `fixtures/f3-viewform.html:1` and `fixtures/f4-viewform.html:1` + +- [ ] **Step 4: Record the URLs** + +Append to `fixtures/README.md`: + +```markdown + +## Test forms + +All require sign-in via Collect email addresses → Verified. + +| Form | Purpose | URL | +|------|---------|-----| +| F1 | 3 required short answers; limit-to-1 OFF (temporarily ON in Task 9) | | +| F3 | 3 sections, mixed question types | | +| F4 | validation rules + file upload | | + +- `f3-viewform.html` — multi-section fixture for parse tests. +- `f4-viewform.html` — validation and file-upload fixture for blocker detection. +``` + +- [ ] **Step 5: Commit** + +```bash +git add fixtures/README.md fixtures/f1-viewform.html fixtures/f3-viewform.html fixtures/f4-viewform.html +git commit -m "Add test forms F3-F4 and page fixtures" +``` + +--- + +### Task 5: Parse form structure + +**Files:** +- Create: `fastform/parse.py` +- Create: `tests/test_parse.py` + +**Interfaces:** +- Consumes: fixtures from Tasks 1 and 4. +- Produces: + - `Question` dataclass: `entry_id: str`, `text: str`, `type: int`, `required: bool`, `options: list[str]` + - `FormSpec` dataclass: `form_id: str`, `action_path: str`, `page_count: int`, `questions: list[Question]`, `hidden: dict[str, str]`; properties `page_history: str`, `blockers: list[Question]` + - `requires_sign_in(html: bytes) -> bool` + - `signed_in_email(html: bytes) -> str | None` + - `hidden_inputs(html: bytes) -> dict[str, str]` + - `parse(html: bytes, url: str) -> FormSpec` + - `form_id_from_url(url: str) -> str` + - `NotParseable` exception + - `BLOCKER_TYPES: frozenset[int]` and `CHOICE_TYPES: frozenset[int]` — Task 6 imports both by name + - `TYPE_SHORT_ANSWER`, `TYPE_PARAGRAPH`, `TYPE_MULTIPLE_CHOICE`, `TYPE_DROPDOWN`, `TYPE_CHECKBOXES`, `TYPE_LINEAR_SCALE`, `TYPE_GRID`, `TYPE_SECTION`, `TYPE_DATE`, `TYPE_TIME`, `TYPE_FILE_UPLOAD` + +**The index positions inside `FB_PUBLIC_LOAD_DATA_` are verified** against the real F1 fixture: `item[1]` = title, `item[3]` = type, `item[4][0][0]` = entry id, `item[4][0][1]` = options, `item[4][0][2]` = required. Step 1 re-confirms them against your own capture. + +**Three facts established empirically during the E1 gate, which this task must encode:** + +1. **A signed-out page still contains the full question structure.** Google Forms does not redirect for a sign-in-required form — it returns HTTP 200 with `FB_PUBLIC_LOAD_DATA_` intact and the inputs disabled. So parsing succeeding proves nothing about being authenticated, and `requires_sign_in()` exists to catch that before a wasted POST. +2. **The signals are HTML attributes, not text.** `data-sign-in-to-continue="true"` appears only when signed out; `data-user-email-address="…"` carries the account identity when signed in and is empty otherwise. Attributes are locale-independent, which matters — the account under test renders in Thai. +3. **The POST needs `token` and `tag`, which live in hidden inputs** and change on every page load. Their values are **HTML-escaped in the page** and must be unescaped, or the POST returns 400 despite having the right field names. + +- [ ] **Step 1: Inspect the real structure first** + +```bash +cd /Users/supa/projects/active/fastform +python3 - <<'PY' +import json, re +html = open("fixtures/f1-viewform.html", "rb").read() +m = re.search(rb"FB_PUBLIC_LOAD_DATA_\s*=\s*(\[.*?\]);\s*", html, re.S) +print("matched:", bool(m)) +data = json.loads(m.group(1)) +print("top-level length:", len(data)) +items = data[1][1] +print("item count:", len(items)) +for it in items: + print(json.dumps(it)[:220]) +PY +``` + +Expected: three items printed, each containing one of your F1 question texts, and each containing a nine- or ten-digit number that matches an `entry.*` ID from Task 3 Step 1. + +Note for each item which position holds the title, which holds the type, and where the entry ID and required flag sit. The code below assumes `item[1]` = title, `item[3]` = type, `item[4][0][0]` = entry id, `item[4][0][1]` = options, `item[4][0][2]` = required. + +- [ ] **Step 2: Write the failing test** + +Create `tests/test_parse.py`: + +```python +import unittest +from pathlib import Path + +from fastform import parse + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" + +F1_URL = "https://docs.google.com/forms/d/e/FAKEID123/viewform" + + +class TestFormIdFromUrl(unittest.TestCase): + def test_extracts_published_id(self): + self.assertEqual( + parse.form_id_from_url( + "https://docs.google.com/forms/d/e/1FAIpQLSabc/viewform?usp=sf_link" + ), + "1FAIpQLSabc", + ) + + def test_rejects_non_form_url(self): + with self.assertRaises(ValueError): + parse.form_id_from_url("https://example.com/hello") + + +class TestParseF1(unittest.TestCase): + @classmethod + def setUpClass(cls): + html = (FIXTURES / "f1-viewform.html").read_bytes() + cls.spec = parse.parse(html, F1_URL) + + def test_finds_three_questions(self): + self.assertEqual(len(self.spec.questions), 3) + + def test_question_text_is_readable(self): + texts = [q.text for q in self.spec.questions] + self.assertIn("What is your full name?", texts) + + def test_entry_ids_are_numeric_strings(self): + for q in self.spec.questions: + self.assertTrue(q.entry_id.isdigit(), q.entry_id) + + def test_all_required(self): + self.assertTrue(all(q.required for q in self.spec.questions)) + + def test_single_page(self): + self.assertEqual(self.spec.page_count, 1) + self.assertEqual(self.spec.page_history, "0") + + def test_action_path(self): + self.assertEqual( + self.spec.action_path, "/forms/d/e/FAKEID123/formResponse" + ) + + def test_no_blockers(self): + self.assertEqual(self.spec.blockers, []) + + def test_hidden_inputs_carry_token_and_tag(self): + self.assertIn("token", self.spec.hidden) + self.assertIn("tag", self.spec.hidden) + self.assertTrue(self.spec.hidden["token"]) + + def test_hidden_values_are_html_unescaped(self): + # partialResponse holds JSON, so it contains quotes that the page + # escapes as ". Leaving them escaped 400s the POST. + self.assertNotIn(""", self.spec.hidden.get("partialResponse", "")) + + +class TestSignInDetection(unittest.TestCase): + """Google Forms returns 200 for a signed-out sign-in-required form, with the + question structure fully present. Only an attribute distinguishes the two.""" + + SIGNED_OUT = b'
' + SIGNED_IN = b'
' + + def test_detects_signed_out(self): + self.assertTrue(parse.requires_sign_in(self.SIGNED_OUT)) + + def test_detects_signed_in(self): + self.assertFalse(parse.requires_sign_in(self.SIGNED_IN)) + + def test_extracts_email_when_signed_in(self): + self.assertEqual( + parse.signed_in_email(self.SIGNED_IN), "someone@example.com" + ) + + def test_no_email_when_signed_out(self): + self.assertIsNone(parse.signed_in_email(self.SIGNED_OUT)) + + +class TestHiddenInputs(unittest.TestCase): + def test_unescapes_values(self): + html = b'' + self.assertEqual(parse.hidden_inputs(html), {"x": 'a"b&c'}) + + def test_ignores_non_hidden_inputs(self): + html = b'' + self.assertEqual(parse.hidden_inputs(html), {}) + + +class TestParseF3(unittest.TestCase): + @classmethod + def setUpClass(cls): + html = (FIXTURES / "f3-viewform.html").read_bytes() + cls.spec = parse.parse(html, F1_URL) + + def test_three_pages(self): + self.assertEqual(self.spec.page_count, 3) + self.assertEqual(self.spec.page_history, "0,1,2") + + def test_choice_options_captured(self): + city = next(q for q in self.spec.questions if "city" in q.text.lower()) + self.assertIn("Bangkok", city.options) + self.assertIn("Chiang Mai", city.options) + + def test_optional_question_detected(self): + optional = [q for q in self.spec.questions if not q.required] + self.assertTrue(optional, "expected at least one optional question") + + +class TestParseF4(unittest.TestCase): + @classmethod + def setUpClass(cls): + html = (FIXTURES / "f4-viewform.html").read_bytes() + cls.spec = parse.parse(html, F1_URL) + + def test_file_upload_is_a_blocker(self): + self.assertEqual(len(self.spec.blockers), 1) + self.assertEqual(self.spec.blockers[0].type, parse.TYPE_FILE_UPLOAD) + + +class TestParseFailures(unittest.TestCase): + def test_missing_load_data_raises(self): + with self.assertRaises(parse.NotParseable): + parse.parse(b"form is closed", F1_URL) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 3: Run the test to verify it fails** + +```bash +python3 -m unittest tests.test_parse -v +``` + +Expected: `ModuleNotFoundError: No module named 'fastform.parse'` + +- [ ] **Step 4: Write the implementation** + +Create `fastform/parse.py`: + +```python +"""Turn a Google Form page into a FormSpec. + +Everything comes out of the FB_PUBLIC_LOAD_DATA_ blob embedded in the page. +A closed form does not include that blob, which is why NotParseable exists and +why the caller has to treat it as a distinct state rather than an error. +""" + +import html as html_mod +import json +import re +from dataclasses import dataclass, field + +TYPE_SHORT_ANSWER = 0 +TYPE_PARAGRAPH = 1 +TYPE_MULTIPLE_CHOICE = 2 +TYPE_DROPDOWN = 3 +TYPE_CHECKBOXES = 4 +TYPE_LINEAR_SCALE = 5 +TYPE_GRID = 7 +TYPE_SECTION = 8 +TYPE_DATE = 9 +TYPE_TIME = 10 +TYPE_FILE_UPLOAD = 13 + +#: Types the direct-POST path cannot satisfy. Hitting one means browser handoff. +BLOCKER_TYPES = frozenset({TYPE_FILE_UPLOAD, TYPE_GRID}) + +#: Types whose value must match one of the offered options. +CHOICE_TYPES = frozenset({TYPE_MULTIPLE_CHOICE, TYPE_DROPDOWN, TYPE_CHECKBOXES}) + +_LOAD_DATA_RE = re.compile( + rb"FB_PUBLIC_LOAD_DATA_\s*=\s*(\[.*?\]);\s*", re.S +) +_FORM_ID_RE = re.compile(r"/forms/d/e/([^/]+)/") +_HIDDEN_RE = re.compile( + rb']+type="hidden"[^>]+name="([^"]+)"[^>]+value="([^"]*)"' +) +#: Present only when the viewer is signed out of a sign-in-required form. +_SIGN_IN_RE = re.compile(rb'data-sign-in-to-continue="true"') +_VIEWER_EMAIL_RE = re.compile(rb'data-user-email-address="([^"]*)"') + + +class NotParseable(Exception): + """The page carries no form structure — closed, deleted, or not a form.""" + + +@dataclass +class Question: + entry_id: str + text: str + type: int + required: bool + options: list[str] = field(default_factory=list) + + +@dataclass +class FormSpec: + form_id: str + action_path: str + page_count: int + questions: list[Question] + #: Hidden inputs from the page, already HTML-unescaped. Carries `token` and + #: `tag`, both of which the POST is rejected without, and both of which + #: change on every page load. + hidden: dict[str, str] = field(default_factory=dict) + + @property + def page_history(self) -> str: + """The pageHistory value that satisfies every section in one POST.""" + return ",".join(str(i) for i in range(self.page_count)) + + @property + def blockers(self) -> list[Question]: + return [q for q in self.questions if q.type in BLOCKER_TYPES] + + +def form_id_from_url(url: str) -> str: + match = _FORM_ID_RE.search(url) + if not match: + raise ValueError(f"not a published Google Form URL: {url}") + return match.group(1) + + +def extract_load_data(html: bytes) -> list: + match = _LOAD_DATA_RE.search(html) + if not match: + raise NotParseable("FB_PUBLIC_LOAD_DATA_ not found") + return json.loads(match.group(1)) + + +def requires_sign_in(html: bytes) -> bool: + """True when this page is a sign-in-required form viewed while signed out. + + Google Forms does not redirect in this case — it serves HTTP 200 with the + whole question structure present and the inputs disabled. Parsing therefore + succeeds on a signed-out page, so this check is the only thing standing + between a dead cookie and a wasted POST. + """ + return bool(_SIGN_IN_RE.search(html)) + + +def signed_in_email(html: bytes) -> str | None: + """The viewing account's address, or None when signed out. + + This is an HTML attribute Google populates from the session, so it is a far + stronger signal than pattern-matching the body for anything email-shaped. + """ + match = _VIEWER_EMAIL_RE.search(html) + if not match: + return None + value = match.group(1).decode("utf-8", "replace") + return value or None + + +def hidden_inputs(html: bytes) -> dict[str, str]: + """Every hidden form input, HTML-unescaped. + + Unescaping is not optional: `partialResponse` holds JSON whose quotes arrive + as `"`, and re-encoding the escaped form gets the POST rejected with a + 400 even when every field name is correct. + """ + return { + name.decode(): html_mod.unescape(value.decode()) + for name, value in _HIDDEN_RE.findall(html) + } + + +def _items(load_data: list) -> list: + try: + return load_data[1][1] or [] + except (IndexError, TypeError) as exc: + raise NotParseable(f"unexpected load data shape: {exc}") from exc + + +def _options(raw_options) -> list[str]: + out = [] + for opt in raw_options or []: + if opt and opt[0]: + out.append(str(opt[0])) + return out + + +def parse(html: bytes, url: str) -> FormSpec: + load_data = extract_load_data(html) + form_id = form_id_from_url(url) + + questions: list[Question] = [] + page_count = 1 + + for item in _items(load_data): + item_type = item[3] + if item_type == TYPE_SECTION: + page_count += 1 + continue + + title = item[1] or "" + for entry in item[4] or []: + questions.append( + Question( + entry_id=str(entry[0]), + text=title, + type=item_type, + required=bool(entry[2]), + options=_options(entry[1]), + ) + ) + + return FormSpec( + form_id=form_id, + action_path=f"/forms/d/e/{form_id}/formResponse", + page_count=page_count, + questions=questions, + hidden=hidden_inputs(html), + ) +``` + +- [ ] **Step 5: Run the test to verify it passes** + +```bash +python3 -m unittest tests.test_parse -v +``` + +Expected: `OK`. + +If index assumptions were wrong you will see failures naming exactly what broke — a question count of 0 means `item[3]`/`item[4]` are misplaced; garbled `text` means `item[1]` is not the title. Adjust the constants in `parse()` and re-run. + +- [ ] **Step 6: Commit** + +```bash +git add fastform/parse.py tests/test_parse.py +git commit -m "Add form structure parser" +``` + +--- + +### Task 6: Match answers to questions + +**Files:** +- Create: `fastform/match.py` +- Create: `tests/test_match.py` + +**Interfaces:** +- Consumes: `Question`, `FormSpec`, `BLOCKER_TYPES`, `CHOICE_TYPES` from `fastform.parse`. +- Produces: + - `FilledForm` dataclass: `pairs: list[tuple[str, str]]`, `gaps: list[Question]`, `blockers: list[Question]`, `confidence: float` + - `load_bank(path: Path | None = None) -> list[dict]` + - `normalize(text: str) -> str` + - `score_question(question, entry) -> float` + - `fill(spec: FormSpec, bank: list[dict]) -> FilledForm` + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_match.py`: + +```python +import unittest + +from fastform import match +from fastform.parse import ( + TYPE_FILE_UPLOAD, + TYPE_MULTIPLE_CHOICE, + TYPE_SHORT_ANSWER, + FormSpec, + Question, +) + + +def spec(*questions): + return FormSpec( + form_id="X", action_path="/p", page_count=1, questions=list(questions) + ) + + +BANK = [ + {"match": ["full name", "your name"], "value": "Test Person"}, + {"match": ["email address"], "regex": "e-?mail", "value": "test@example.com"}, + {"match": ["which city"], "value": "Bangkok"}, +] + + +class TestNormalize(unittest.TestCase): + def test_casefolds_and_strips_punctuation(self): + self.assertEqual(match.normalize("What is your Name?"), "what is your name") + + def test_collapses_whitespace(self): + self.assertEqual(match.normalize("a b\n c"), "a b c") + + +class TestFill(unittest.TestCase): + def test_exact_text_match_fills(self): + s = spec(Question("111", "full name", TYPE_SHORT_ANSWER, True)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, [("111", "Test Person")]) + self.assertEqual(filled.gaps, []) + + def test_regex_wins_and_scores_one(self): + s = spec(Question("222", "Your E-Mail", TYPE_SHORT_ANSWER, True)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, [("222", "test@example.com")]) + self.assertEqual(filled.confidence, 1.0) + + def test_unmatched_required_becomes_a_gap(self): + s = spec(Question("333", "Favourite dinosaur", TYPE_SHORT_ANSWER, True)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, []) + self.assertEqual([q.entry_id for q in filled.gaps], ["333"]) + + def test_unmatched_optional_is_omitted_not_a_gap(self): + s = spec(Question("444", "Favourite dinosaur", TYPE_SHORT_ANSWER, False)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, []) + self.assertEqual(filled.gaps, []) + + def test_choice_value_must_exist_in_options(self): + s = spec( + Question( + "555", + "Which city?", + TYPE_MULTIPLE_CHOICE, + True, + options=["Bangkok Metropolitan Region", "Phuket"], + ) + ) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, []) + self.assertEqual([q.entry_id for q in filled.gaps], ["555"]) + + def test_choice_value_present_in_options_fills(self): + s = spec( + Question( + "666", + "Which city?", + TYPE_MULTIPLE_CHOICE, + True, + options=["Bangkok", "Phuket"], + ) + ) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, [("666", "Bangkok")]) + + def test_blockers_are_reported_separately(self): + s = spec(Question("777", "Upload", TYPE_FILE_UPLOAD, False)) + filled = match.fill(s, BANK) + self.assertEqual([q.entry_id for q in filled.blockers], ["777"]) + + def test_confidence_is_min_over_required_questions(self): + s = spec( + Question("111", "full name", TYPE_SHORT_ANSWER, True), + Question("333", "Favourite dinosaur", TYPE_SHORT_ANSWER, True), + ) + filled = match.fill(s, BANK) + self.assertEqual(filled.confidence, 0.0) + + def test_confidence_is_one_when_no_required_questions(self): + s = spec(Question("444", "Anything", TYPE_SHORT_ANSWER, False)) + self.assertEqual(match.fill(s, BANK).confidence, 1.0) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +python3 -m unittest tests.test_match -v +``` + +Expected: `ModuleNotFoundError: No module named 'fastform.match'` + +- [ ] **Step 3: Write the implementation** + +Create `fastform/match.py`: + +```python +"""Match prepared answers to a form's questions by text, never by position. + +Position matching breaks the moment a question is added or reordered. Text +matching degrades gracefully: a near miss still scores, and the score is what +decides whether firing is safe or the operator has to look. +""" + +import difflib +import re +import tomllib +import unicodedata +from dataclasses import dataclass, field +from pathlib import Path + +from fastform.parse import BLOCKER_TYPES, CHOICE_TYPES, FormSpec, Question + +DEFAULT_BANK_PATH = Path(__file__).resolve().parent.parent / "config" / "answers.toml" + +#: Below this, a fuzzy hit is treated as no match at all. +FUZZY_FLOOR = 0.6 + +_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE) + + +@dataclass +class FilledForm: + pairs: list[tuple[str, str]] = field(default_factory=list) + gaps: list[Question] = field(default_factory=list) + blockers: list[Question] = field(default_factory=list) + confidence: float = 1.0 + + +def load_bank(path: Path | None = None) -> list[dict]: + path = path or DEFAULT_BANK_PATH + with path.open("rb") as fh: + return tomllib.load(fh).get("answer", []) + + +def normalize(text: str) -> str: + text = unicodedata.normalize("NFKC", text).casefold() + text = _PUNCT_RE.sub(" ", text) + return " ".join(text.split()) + + +def score_question(question: Question, entry: dict) -> float: + """How well does this bank entry match this question? 0.0 means no match.""" + pattern = entry.get("regex") + if pattern and re.search(pattern, question.text, re.IGNORECASE): + return 1.0 + + target = normalize(question.text) + target_tokens = set(target.split()) + best = 0.0 + + for phrase in entry.get("match", []): + candidate = normalize(phrase) + if not candidate: + continue + if candidate == target: + best = max(best, 0.95) + continue + if set(candidate.split()) <= target_tokens: + best = max(best, 0.8) + continue + ratio = difflib.SequenceMatcher(None, candidate, target).ratio() + if ratio >= FUZZY_FLOOR: + best = max(best, ratio) + + return best + + +def _resolve_choice(value: str, options: list[str]) -> str | None: + """A choice answer must be one of the offered options, or it will be rejected.""" + if not options: + return value + normalized = {normalize(o): o for o in options} + return normalized.get(normalize(value)) + + +def fill(spec: FormSpec, bank: list[dict]) -> FilledForm: + filled = FilledForm() + scores: list[float] = [] + + for question in spec.questions: + if question.type in BLOCKER_TYPES: + filled.blockers.append(question) + continue + + best_score, best_value = 0.0, None + for entry in bank: + score = score_question(question, entry) + if score > best_score: + best_score, best_value = score, entry["value"] + + if best_value is not None and question.type in CHOICE_TYPES: + best_value = _resolve_choice(best_value, question.options) + if best_value is None: + best_score = 0.0 + + if best_value is not None and best_score > 0.0: + filled.pairs.append((question.entry_id, best_value)) + if question.required: + scores.append(best_score) + elif question.required: + filled.gaps.append(question) + scores.append(0.0) + + filled.confidence = min(scores) if scores else 1.0 + return filled +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +python3 -m unittest tests.test_match -v +``` + +Expected: `Ran 12 tests` … `OK` + +- [ ] **Step 5: Commit** + +```bash +git add fastform/match.py tests/test_match.py +git commit -m "Add text-based answer matching with confidence scoring" +``` + +--- + +### Task 7: Compile the request + +**Files:** +- Create: `fastform/compile.py` +- Create: `tests/test_compile.py` + +**Interfaces:** +- Consumes: `FormSpec` from `fastform.parse`, `FilledForm` from `fastform.match`, `USER_AGENT` from `fastform.cookies`. +- Produces: + - `RequestBlob` dataclass: `host: str`, `path: str`, `body: str`, `headers: dict[str, str]` + - `new_fbzx() -> str` + - `build(spec, filled, cookie, email=None, fbzx=None) -> RequestBlob` + - `MissingToken` exception + +**Established empirically during the E1 gate — the payload the plan originally specified returns HTTP 400.** Measured against F1 by dropping one field at a time: + +| Field | Verdict | Source | +|---|---|---| +| `token` | **Required.** Without it: 400 | `spec.hidden["token"]`, rotates per page load | +| `tag` | **Required.** Without it: 400 | `spec.hidden["tag"]` | +| `emailAddress` | **Required** when the form collects the responder's email | caller supplies; `parse.signed_in_email()` reads it from the page | +| `partialResponse`, `dlut`, `hud` | Optional — dropping them still records | — | + +`token` is **not** a reCAPTCHA value: the page carries no recaptcha, grecaptcha, or botguard reference. It is a plain hidden input, which is why scraping it works and why the design survives. + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_compile.py`: + +```python +import unittest +import urllib.parse + +from fastform import compile as compile_mod +from fastform.match import FilledForm +from fastform.parse import FormSpec + + +def spec(pages=1, hidden=None): + return FormSpec( + form_id="ABC", action_path="/forms/d/e/ABC/formResponse", + page_count=pages, questions=[], + hidden=hidden if hidden is not None else {"token": "TOK", "tag": "TAG"}, + ) + + +class TestNewFbzx(unittest.TestCase): + def test_is_a_signed_integer_string(self): + value = compile_mod.new_fbzx() + int(value) + + def test_differs_between_calls(self): + self.assertNotEqual(compile_mod.new_fbzx(), compile_mod.new_fbzx()) + + +class TestBuild(unittest.TestCase): + def _fields(self, blob): + return dict(urllib.parse.parse_qsl(blob.body)) + + def test_entry_pairs_are_prefixed(self): + filled = FilledForm(pairs=[("123", "Alice")]) + blob = compile_mod.build(spec(), filled, "SID=x", fbzx="-1") + self.assertEqual(self._fields(blob)["entry.123"], "Alice") + + def test_includes_fvv_and_timestamp(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + fields = self._fields(blob) + self.assertEqual(fields["fvv"], "1") + self.assertEqual(fields["submissionTimestamp"], "-1") + + def test_page_history_covers_every_section(self): + blob = compile_mod.build(spec(pages=3), FilledForm(), "SID=x", fbzx="-1") + self.assertEqual(self._fields(blob)["pageHistory"], "0,1,2") + + def test_cookie_and_content_type_headers(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=abc", fbzx="-1") + self.assertEqual(blob.headers["Cookie"], "SID=abc") + self.assertEqual( + blob.headers["Content-Type"], "application/x-www-form-urlencoded" + ) + + def test_referer_points_at_the_viewform(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + self.assertEqual( + blob.headers["Referer"], + "https://docs.google.com/forms/d/e/ABC/viewform", + ) + + def test_unicode_values_survive_encoding(self): + filled = FilledForm(pairs=[("1", "กรุงเทพ")]) + blob = compile_mod.build(spec(), filled, "SID=x", fbzx="-1") + self.assertEqual(self._fields(blob)["entry.1"], "กรุงเทพ") + + def test_generates_fbzx_when_not_supplied(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x") + int(self._fields(blob)["fbzx"]) + + def test_content_length_matches_encoded_body(self): + filled = FilledForm(pairs=[("1", "กรุงเทพ")]) + blob = compile_mod.build(spec(), filled, "SID=x", fbzx="-1") + self.assertEqual( + blob.headers["Content-Length"], + str(len(blob.body.encode("utf-8"))), + ) + + def test_carries_token_and_tag_from_the_page(self): + # Measured: dropping either gets the POST rejected with 400. + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + fields = self._fields(blob) + self.assertEqual(fields["token"], "TOK") + self.assertEqual(fields["tag"], "TAG") + + def test_missing_token_raises_rather_than_posting_garbage(self): + with self.assertRaises(compile_mod.MissingToken): + compile_mod.build( + spec(hidden={"tag": "TAG"}), FilledForm(), "SID=x", fbzx="-1" + ) + + def test_missing_tag_raises(self): + with self.assertRaises(compile_mod.MissingToken): + compile_mod.build( + spec(hidden={"token": "TOK"}), FilledForm(), "SID=x", fbzx="-1" + ) + + def test_email_included_when_supplied(self): + blob = compile_mod.build( + spec(), FilledForm(), "SID=x", email="a@b.test", fbzx="-1" + ) + self.assertEqual(self._fields(blob)["emailAddress"], "a@b.test") + + def test_email_omitted_when_not_supplied(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + self.assertNotIn("emailAddress", self._fields(blob)) + + def test_page_fbzx_preferred_over_generated(self): + s = spec(hidden={"token": "TOK", "tag": "TAG", "fbzx": "42"}) + blob = compile_mod.build(s, FilledForm(), "SID=x") + self.assertEqual(self._fields(blob)["fbzx"], "42") + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +python3 -m unittest tests.test_compile -v +``` + +Expected: `ModuleNotFoundError: No module named 'fastform.compile'` + +- [ ] **Step 3: Write the implementation** + +Create `fastform/compile.py`: + +```python +"""Turn a filled form into a ready-to-send request. + +Everything expensive happens here, at arm time. Sending is then a socket write +with nothing left to compute — which is the whole point of separating this from +transport. +""" + +import random +import urllib.parse +from dataclasses import dataclass + +from fastform.cookies import USER_AGENT +from fastform.match import FilledForm +from fastform.parse import FormSpec + +HOST = "docs.google.com" + +#: Hidden inputs the server rejects the POST without. Established by dropping +#: each field in turn against a real form: omitting either yields HTTP 400. +REQUIRED_HIDDEN = ("token", "tag") + + +class MissingToken(Exception): + """The page did not yield `token`/`tag`, so no POST can succeed. + + Raised rather than sending a request that is certain to 400 — the failure is + better surfaced at arm time than as a mystery rejection mid-run. + """ + + +@dataclass +class RequestBlob: + host: str + path: str + body: str + headers: dict[str, str] + + +def new_fbzx() -> str: + """A fresh anti-duplicate token, in the same shape Google's own pages emit.""" + return str(random.randint(-(2**63), 2**63 - 1)) + + +def build( + spec: FormSpec, + filled: FilledForm, + cookie: str, + email: str | None = None, + fbzx: str | None = None, +) -> RequestBlob: + missing = [k for k in REQUIRED_HIDDEN if not spec.hidden.get(k)] + if missing: + raise MissingToken( + f"page yielded no {', '.join(missing)} — the POST would be rejected" + ) + + fields: list[tuple[str, str]] = [ + (f"entry.{entry_id}", value) for entry_id, value in filled.pairs + ] + if email: + fields.append(("emailAddress", email)) + fields.append(("fvv", "1")) + # Prefer the page's own fbzx; it is already paired with this token. + fields.append(("fbzx", fbzx or spec.hidden.get("fbzx") or new_fbzx())) + fields.append(("pageHistory", spec.page_history)) + for key in REQUIRED_HIDDEN: + fields.append((key, spec.hidden[key])) + fields.append(("submissionTimestamp", "-1")) + + body = urllib.parse.urlencode(fields, encoding="utf-8") + viewform = f"https://{HOST}/forms/d/e/{spec.form_id}/viewform" + + return RequestBlob( + host=HOST, + path=spec.action_path, + body=body, + headers={ + "Cookie": cookie, + "User-Agent": USER_AGENT, + "Content-Type": "application/x-www-form-urlencoded", + "Content-Length": str(len(body.encode("utf-8"))), + "Referer": viewform, + "Origin": f"https://{HOST}", + }, + ) +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +python3 -m unittest tests.test_compile -v +``` + +Expected: `Ran 10 tests` … `OK` + +- [ ] **Step 5: Commit** + +```bash +git add fastform/compile.py tests/test_compile.py +git commit -m "Add request compiler" +``` + +--- + +### Task 8: Dry-run tool and the cURL diff + +**Files:** +- Create: `tools/dryrun.py` +- Create: `tools/curl_diff.py` + +**Interfaces:** +- Consumes: `cookies`, `parse`, `match`, `compile` from Tasks 2, 5, 6, 7. +- Produces: `tools/dryrun.py --url ` printing the request that would be sent. No later task imports these. + +This is the highest-value verification in the plan: it catches every missing field at once by comparing against a request Google actually accepted. + +- [ ] **Step 1: Write the dry-run tool** + +Create `tools/dryrun.py`: + +```python +"""Build the request for a form and print it. Sends nothing. + +Usage: + python3 tools/dryrun.py --url +""" + +import argparse +import http.client +import sys +import urllib.parse +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from fastform import compile as compile_mod # noqa: E402 +from fastform import cookies, match, parse # noqa: E402 + + +def fetch(url: str, cookie: str) -> bytes: + parts = urllib.parse.urlsplit(url) + conn = http.client.HTTPSConnection(parts.netloc, timeout=15) + try: + conn.request( + "GET", + parts.path + ("?" + parts.query if parts.query else ""), + headers={"Cookie": cookie, "User-Agent": cookies.USER_AGENT}, + ) + resp = conn.getresponse() + return resp.read() + finally: + conn.close() + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--url", required=True) + args = ap.parse_args() + + cookie = cookies.load() + html = fetch(args.url, cookie) + + try: + spec = parse.parse(html, args.url) + except parse.NotParseable as exc: + print(f"NOT PARSEABLE: {exc}") + print("The form is closed, deleted, or the cookie is not signed in.") + return 2 + + filled = match.fill(spec, match.load_bank()) + blob = compile_mod.build(spec, filled, cookie) + + print(f"form_id {spec.form_id}") + print(f"pages {spec.page_count} (pageHistory={spec.page_history})") + print(f"questions {len(spec.questions)}") + print(f"confidence {filled.confidence:.2f}") + print() + + for question in spec.questions: + value = dict(filled.pairs).get(question.entry_id) + flag = "REQ" if question.required else "opt" + status = value if value is not None else "— unfilled —" + print(f" [{flag}] entry.{question.entry_id:<12} {question.text[:44]:<46} {status}") + + if filled.gaps: + print(f"\nGAPS ({len(filled.gaps)}) — required and unmatched:") + for question in filled.gaps: + print(f" entry.{question.entry_id} {question.text}") + + if filled.blockers: + print(f"\nBLOCKERS ({len(filled.blockers)}) — direct POST cannot satisfy these:") + for question in filled.blockers: + print(f" entry.{question.entry_id} type={question.type} {question.text}") + + print(f"\nPOST https://{blob.host}{blob.path}") + for key, value in blob.headers.items(): + shown = "" if key == "Cookie" else value + print(f" {key}: {shown}") + print(f"\nbody: {blob.body}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **Step 2: Run it against F1** + +```bash +cd /Users/supa/projects/active/fastform +python3 tools/dryrun.py --url '' +``` + +Expected: three questions listed, all filled, `confidence 0.80` or higher, no gaps, no blockers, and a body containing three `entry.*` fields plus `fvv`, `fbzx`, `pageHistory`, `submissionTimestamp`. + +- [ ] **Step 3: Run it against F4 to confirm blocker detection** + +```bash +python3 tools/dryrun.py --url '' +``` + +Expected: a `BLOCKERS (1)` section naming the file upload question with `type=13`. + +- [ ] **Step 4: Write the cURL diff tool** + +Create `tools/curl_diff.py`: + +```python +"""Compare our generated POST body against a real one captured from DevTools. + +Any field Google's own page sent that we do not is a candidate explanation for +a rejected submission. + +Usage: + python3 tools/curl_diff.py --curl fixtures/f1-manual-submit.curl.txt --url +""" + +import argparse +import re +import sys +import urllib.parse +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from fastform import compile as compile_mod # noqa: E402 +from fastform import cookies, match, parse # noqa: E402 +from tools.dryrun import fetch # noqa: E402 + +_DATA_RE = re.compile(r"--data(?:-raw|-urlencode)?\s+\$?'(.*?)'", re.S) + + +def curl_fields(text: str) -> dict[str, str]: + chunks = _DATA_RE.findall(text) + if not chunks: + raise SystemExit("no --data section found in the cURL file") + return dict(urllib.parse.parse_qsl("&".join(chunks), keep_blank_values=True)) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--curl", required=True) + ap.add_argument("--url", required=True) + args = ap.parse_args() + + theirs = curl_fields(Path(args.curl).read_text(encoding="utf-8")) + + cookie = cookies.load() + spec = parse.parse(fetch(args.url, cookie), args.url) + filled = match.fill(spec, match.load_bank()) + ours = dict( + urllib.parse.parse_qsl( + compile_mod.build(spec, filled, cookie).body, keep_blank_values=True + ) + ) + + missing = sorted(set(theirs) - set(ours)) + extra = sorted(set(ours) - set(theirs)) + + print(f"chrome sent {len(theirs)} fields, we send {len(ours)}\n") + + if missing: + print("MISSING — Chrome sent these and we do not:") + for key in missing: + print(f" {key} = {theirs[key]!r}") + else: + print("MISSING — none") + + print() + if extra: + print("EXTRA — we send these and Chrome did not:") + for key in extra: + print(f" {key} = {ours[key]!r}") + else: + print("EXTRA — none") + + return 1 if missing else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **Step 5: Run the diff** + +```bash +touch tools/__init__.py +python3 tools/curl_diff.py --curl fixtures/f1-manual-submit.curl.txt --url '' +``` + +Expected: `MISSING — none`. + +If fields are listed as missing, add them to `fastform/compile.py`'s `fields` list, add a test in `tests/test_compile.py` asserting each new field is present, and re-run until the diff is clean. Common additions are `entry._sentinel` fields for checkbox questions and a `draftResponse` parameter. + +- [ ] **Step 6: Commit** + +```bash +git add tools/dryrun.py tools/curl_diff.py tools/__init__.py +git commit -m "Add dry-run and cURL diff verification tools" +``` + +--- + +### Task 9: Classify responses + +**Files:** +- Create: `fastform/classify.py` +- Create: `tests/test_classify.py` +- Create: `tools/capture_responses.py` + +**Interfaces:** +- Consumes: `cookies`, `parse`, `match`, `compile` from earlier tasks. +- Produces: + - `Outcome` enum: `RECORDED`, `REJECTED`, `CLOSED`, `ALREADY_RESPONDED`, `AUTH_FAIL`, `UNKNOWN` + - `classify(status: int, location: str | None, body: bytes) -> Outcome` + +The fixtures are the specification here. Capture them first, look at them, then make the classifier satisfy them. + +- [ ] **Step 1: Write the capture tool** + +Create `tools/capture_responses.py`: + +```python +"""Capture real response bodies for each outcome, to use as classifier fixtures. + +Usage: + python3 tools/capture_responses.py --url --name success +""" + +import argparse +import http.client +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from fastform import compile as compile_mod # noqa: E402 +from fastform import cookies, match, parse # noqa: E402 +from tools.dryrun import fetch # noqa: E402 + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--url", required=True) + ap.add_argument("--name", required=True, help="fixture suffix, e.g. success") + ap.add_argument( + "--drop-one-required", + action="store_true", + help="omit a single required field, to provoke a validation rejection", + ) + args = ap.parse_args() + + cookie = cookies.load() + html = fetch(args.url, cookie) + + try: + spec = parse.parse(html, args.url) + except parse.NotParseable: + FIXTURES.joinpath(f"resp-{args.name}.bin").write_bytes(html) + print(f"form not parseable; saved the GET body as resp-{args.name}.bin") + return 0 + + filled = match.fill(spec, match.load_bank()) + + if args.drop_one_required: + # Drop exactly one required answer rather than emptying the payload. + # An entirely empty POST risks being treated as a draft rather than a + # rejection, which would make the fixture the wrong shape. + required_ids = {q.entry_id for q in spec.questions if q.required} + for index, (entry_id, _) in enumerate(filled.pairs): + if entry_id in required_ids: + dropped = filled.pairs.pop(index) + print(f"dropped required field entry.{dropped[0]}") + break + else: + raise SystemExit("no required field was filled — nothing to drop") + + blob = compile_mod.build(spec, filled, cookie) + conn = http.client.HTTPSConnection(blob.host, timeout=15) + try: + conn.request("POST", blob.path, body=blob.body, headers=blob.headers) + resp = conn.getresponse() + body = resp.read() + finally: + conn.close() + + out = FIXTURES / f"resp-{args.name}.bin" + out.write_bytes(body) + print(f"status={resp.status} location={resp.getheader('Location')} bytes={len(body)}") + print(f"saved {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **Step 2: Capture all five fixtures** + +All four go against F1, in this order. Two of them need a Forms setting toggled +first — do them in sequence, not out of order. + +```bash +cd /Users/supa/projects/active/fastform + +# 1. success — F1 has limit-to-1 off, so this is repeatable +python3 tools/capture_responses.py --url '' --name success + +# 2. rejected — one required field omitted +python3 tools/capture_responses.py --url '' --name rejected --drop-one-required +``` + +**3. closed.** In the Forms UI, turn **OFF** F1's "Accepting responses". Then: + +```bash +python3 tools/capture_responses.py --url '' --name closed +``` + +The tool takes its not-parseable branch here and saves the GET body — that page +*is* the closed-form page, which is what the classifier needs. Now turn +**Accepting responses back ON.** + +**4. already-responded.** In the Forms UI, turn **ON** F1's "Limit to 1 response". +The rows already in the sheet make your account count as having responded. Then: + +```bash +python3 tools/capture_responses.py --url '' --name already +``` + +This also takes the not-parseable branch, because an already-responded form does +not render its questions. Now turn **Limit to 1 response back OFF**, or Task 11's +end-to-end check will fail. + +There is deliberately no `authfail` fixture. `AUTH_FAIL` is detected from the +redirect status and `Location` header, neither of which a saved body preserves — +the fixture would be unusable, and the structural unit test in Step 4 already +covers it. Capturing it would also mean temporarily overwriting your real +`config/cookies.txt`, which is a needless risk for an artifact nothing reads. + +Verify all four exist: + +```bash +ls -la fixtures/resp-*.bin +``` + +Expected: `resp-success.bin`, `resp-rejected.bin`, `resp-closed.bin`, `resp-already.bin` (plus `resp-e1.bin` from Task 3). + +- [ ] **Step 3: Inspect what actually distinguishes them** + +```bash +for f in fixtures/resp-*.bin; do + echo "=== $f ($(wc -c < "$f") bytes)" + echo -n " FB_PUBLIC_LOAD_DATA_: "; grep -c FB_PUBLIC_LOAD_DATA_ "$f" || true + echo -n " freebirdFormviewerViewResponseConfirm: "; grep -c freebirdFormviewerViewResponseConfirm "$f" || true +done +``` + +Read the output. Note which markers are present in which fixture — the implementation in Step 5 uses these, and **your values may differ from the ones written there.** Adjust the constants to match what you observe. + +- [ ] **Step 4: Write the failing test** + +Create `tests/test_classify.py`: + +```python +import unittest +from pathlib import Path + +from fastform.classify import Outcome, classify + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" + + +class TestClassifyRealFixtures(unittest.TestCase): + """Ground truth: bodies captured from real submissions to F1. + + resp-success.bin is the case an earlier classifier got wrong (reported + REJECTED on a recorded submission), so it is the most important row here. + """ + + def test_success_is_recorded(self): + body = (FIXTURES / "resp-success.bin").read_bytes() + self.assertEqual(classify(200, None, body), Outcome.RECORDED) + + def test_rejected_is_rejected(self): + body = (FIXTURES / "resp-rejected.bin").read_bytes() + self.assertEqual(classify(400, None, body), Outcome.REJECTED) + + def test_closed_landing_page_is_closed(self): + body = (FIXTURES / "resp-closed.bin").read_bytes() + self.assertEqual(classify(200, None, body), Outcome.CLOSED) + + +class TestClassifyRedirects(unittest.TestCase): + def test_302_to_closedform_is_closed(self): + self.assertEqual( + classify(302, "https://docs.google.com/forms/u/0/d/e/X/closedform", b""), + Outcome.CLOSED, + ) + + def test_302_to_accounts_is_auth_fail(self): + self.assertEqual( + classify(302, "https://accounts.google.com/ServiceLogin", b""), + Outcome.AUTH_FAIL, + ) + + def test_other_redirect_is_unknown(self): + self.assertEqual( + classify(302, "https://docs.google.com/forms/d/e/X/viewform", b""), + Outcome.UNKNOWN, + ) + + +class TestClassifyStructural(unittest.TestCase): + def test_signed_out_200_is_auth_fail_not_rejected(self): + # A signed-out sign-in-required form is served as 200 with the whole + # form. Without the sign-in check this reaches the form-element branch + # and reports REJECTED, sending the operator after a phantom payload bug. + body = ( + b'
' + b"" + b'
' + ) + self.assertEqual(classify(200, None, body), Outcome.AUTH_FAIL) + + def test_form_element_on_200_is_rejected(self): + body = b'' + self.assertEqual(classify(200, None, body), Outcome.REJECTED) + + def test_terminal_page_without_markers_is_unknown(self): + # No form, but no recorded marker either: never guess RECORDED. + self.assertEqual(classify(200, None, b"something else"), Outcome.UNKNOWN) + + def test_server_error_is_unknown(self): + self.assertEqual(classify(500, None, b"oops"), Outcome.UNKNOWN) + + def test_empty_body_is_unknown(self): + self.assertEqual(classify(200, None, b""), Outcome.UNKNOWN) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 5: Write the implementation** + +Create `fastform/classify.py`: + +```python +"""Decide what a response means, from measured fixtures — not guesses. + +Every rule here was derived from real captured responses (see fixtures/README). +The signals are structural and locale-independent, which matters because the +account under test renders Forms in Thai. An earlier version keyed on +`FB_PUBLIC_LOAD_DATA_` reappearing and on `freebird*` CSS classes; both were +wrong — the success page also carries `FB_PUBLIC_LOAD_DATA_`, and those classes +do not exist in current markup. That mistake reported a recorded submission as +REJECTED, which is the worst error available because it invites a retry. +""" + +from enum import Enum +from urllib.parse import urlsplit + +#: The actual element. Present only when the form came back for +#: another attempt, i.e. the submission did NOT record. This is the primary +#: recorded-vs-rejected discriminator. +FORM_ELEMENT = b" bool: + return any(marker in body for marker in markers) + + +def classify(status: int, location: str | None, body: bytes) -> Outcome: + # Redirects carry the signal before any body is read. A closed form 302s to + # /closedform; a dead session 302s to accounts.google.com. + if status in (301, 302, 303, 307, 308): + loc = location or "" + if "accounts.google.com" in loc: + return Outcome.AUTH_FAIL + if urlsplit(loc).path.endswith("/closedform"): + return Outcome.CLOSED + return Outcome.UNKNOWN + + if not body: + return Outcome.UNKNOWN + + # Signed-out sign-in-required form: HTTP 200 with the whole form present. + # Must precede the form-element test below. + if SIGN_IN_MARKER in body: + return Outcome.AUTH_FAIL + + # The closed landing page, if a redirect was followed through to it. + if CLOSED_MARKER in body and LOAD_DATA not in body: + return Outcome.CLOSED + + # A re-rendered form means the submission did not take. Measured rejects come + # back 400, but keying on the form element as well as the status is stronger. + if status == 400 or FORM_ELEMENT in body: + return Outcome.REJECTED + + # Terminal confirmation page: no form element, positive recorded marker. + if _has_any(body, RECORDED_MARKERS): + return Outcome.RECORDED + + return Outcome.UNKNOWN +``` + +- [ ] **Step 6: Run the test** + +```bash +python3 -m unittest tests.test_classify -v +``` + +Expected: `OK`. + +If a fixture maps to the wrong outcome, the markers above do not match your account's pages. Go back to Step 3's output, find a string that is present in exactly one fixture, and use it. **Do not weaken the test to match the code** — the fixtures are ground truth. + +- [ ] **Step 7: Commit** + +```bash +git add fastform/classify.py tests/test_classify.py tools/capture_responses.py fixtures/resp-*.bin +git commit -m "Add response classifier with captured fixtures" +``` + +--- + +### Task 10: Warm connection transport + +**Files:** +- Create: `fastform/transport.py` +- Create: `tests/test_transport.py` + +**Interfaces:** +- Consumes: `USER_AGENT` from `fastform.cookies`, `RequestBlob` from `fastform.compile`. +- Produces: + - `Response` dataclass: `status: int`, `location: str | None`, `body: bytes`, `elapsed_ms: float` + - `WarmConnection` class: `connect()`, `heartbeat()`, `get(path, cookie) -> Response`, `send(blob) -> Response`, `close()` + +**Be clear about what this buys at Phase 4.** A hand-run tool connects on demand, +so the connection is cold on first use and none of the spec's warm-pool latency +win applies yet — that needs a long-running daemon, which is Phase 7. What this +does buy immediately is real: `fire.py` performs its GET and its POST on the +same connection, so the POST costs no handshake at all. One saved TLS setup on +the request that matters. The pool of 2–3 connections and the heartbeat loop are +Phase 7's job; `heartbeat()` exists here because it is small and belongs with +the class it operates on, not because anything in this plan calls it in a loop. + +- [ ] **Step 1: Write the failing test** + +Uses a local `http.server`, so no external network. + +Create `tests/test_transport.py`: + +```python +import http.server +import threading +import unittest + +from fastform.compile import RequestBlob +from fastform.transport import Response, WarmConnection + + +class _Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b"echo:" + body) + + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, *args): + pass + + +class TestWarmConnection(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.server = http.server.HTTPServer(("127.0.0.1", 0), _Handler) + cls.port = cls.server.server_address[1] + cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) + cls.thread.start() + + @classmethod + def tearDownClass(cls): + cls.server.shutdown() + cls.server.server_close() + + def _conn(self): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False) + conn.connect() + self.addCleanup(conn.close) + return conn + + def test_send_returns_body_and_status(self): + conn = self._conn() + blob = RequestBlob( + host="127.0.0.1", path="/x", body="a=1", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + resp = conn.send(blob) + self.assertIsInstance(resp, Response) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.body, b"echo:a=1") + + def test_records_elapsed_time(self): + conn = self._conn() + blob = RequestBlob(host="127.0.0.1", path="/x", body="a=1", headers={}) + self.assertGreater(conn.send(blob).elapsed_ms, 0.0) + + def test_connection_is_reusable(self): + conn = self._conn() + blob = RequestBlob(host="127.0.0.1", path="/x", body="a=1", headers={}) + self.assertEqual(conn.send(blob).status, 200) + self.assertEqual(conn.send(blob).status, 200) + + def test_heartbeat_keeps_it_alive(self): + conn = self._conn() + self.assertTrue(conn.heartbeat("/ping", "SID=x")) + + def test_send_before_connect_raises(self): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False) + blob = RequestBlob(host="127.0.0.1", path="/x", body="", headers={}) + with self.assertRaises(RuntimeError): + conn.send(blob) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +python3 -m unittest tests.test_transport -v +``` + +Expected: `ModuleNotFoundError: No module named 'fastform.transport'` + +- [ ] **Step 3: Write the implementation** + +Create `fastform/transport.py`: + +```python +"""Hold a connection open so sending costs a socket write, not a handshake. + +A cold DNS + TCP + TLS setup to docs.google.com on residential internet is +150-350ms and highly variable — bigger and jitterier than anything else in the +path. Connecting early and heartbeating turns that into zero. +""" + +import http.client +import time +from dataclasses import dataclass + +from fastform.compile import RequestBlob +from fastform.cookies import USER_AGENT + + +@dataclass +class Response: + status: int + location: str | None + body: bytes + elapsed_ms: float + + +class WarmConnection: + """One pre-established HTTP connection, reused across requests.""" + + def __init__( + self, + host: str, + port: int | None = None, + timeout: float = 15.0, + use_tls: bool = True, + ): + self.host = host + self.port = port + self.timeout = timeout + self.use_tls = use_tls + self._conn: http.client.HTTPConnection | None = None + + def connect(self) -> None: + cls = ( + http.client.HTTPSConnection if self.use_tls else http.client.HTTPConnection + ) + self._conn = cls(self.host, port=self.port, timeout=self.timeout) + self._conn.connect() + + def close(self) -> None: + if self._conn is not None: + self._conn.close() + self._conn = None + + def _request(self, method, path, body, headers) -> Response: + if self._conn is None: + raise RuntimeError("connect() must be called before sending") + started = time.perf_counter_ns() + self._conn.request(method, path, body=body, headers=headers) + resp = self._conn.getresponse() + payload = resp.read() # must drain fully or the connection cannot be reused + elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000 + return Response( + status=resp.status, + location=resp.getheader("Location"), + body=payload, + elapsed_ms=elapsed_ms, + ) + + def get(self, path: str, cookie: str) -> Response: + return self._request( + "GET", path, None, {"Cookie": cookie, "User-Agent": USER_AGENT} + ) + + def send(self, blob: RequestBlob) -> Response: + return self._request("POST", blob.path, blob.body, blob.headers) + + def heartbeat(self, path: str, cookie: str) -> bool: + """Keep the socket and any NAT mapping alive. Reconnects on failure.""" + try: + self.get(path, cookie) + return True + except (OSError, http.client.HTTPException): + self.close() + try: + self.connect() + return True + except OSError: + return False +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +python3 -m unittest tests.test_transport -v +``` + +Expected: `Ran 5 tests` … `OK` + +- [ ] **Step 5: Commit** + +```bash +git add fastform/transport.py tests/test_transport.py +git commit -m "Add warm connection transport" +``` + +--- + +### Task 11: The single-shot sender + +**Files:** +- Create: `fastform/fire.py` +- Create: `tests/test_fire.py` +- Modify: `README.md` + +**Interfaces:** +- Consumes: every module from Tasks 2, 5, 6, 7, 9, 10. +- Produces: `python3 -m fastform.fire --url ` — the deliverable for Phases 0–4. + +**This sender fires exactly once.** No loop, no resend path. It cannot double-submit because there is no code that sends twice. Retry discipline arrives in spec Phase 6, after calibration measures what is actually safe. Until then a timed-out send is a loss and the operator falls back to the browser. + +- [ ] **Step 1: Write the failing test** + +Only the pure decision logic is unit-tested; the network path is verified manually in Step 5. + +Create `tests/test_fire.py`: + +```python +import unittest + +from fastform.fire import ExitCode, decide_exit +from fastform.classify import Outcome + + +class TestDecideExit(unittest.TestCase): + def test_recorded_is_success(self): + self.assertEqual(decide_exit(Outcome.RECORDED), ExitCode.OK) + + def test_already_responded_is_success(self): + self.assertEqual(decide_exit(Outcome.ALREADY_RESPONDED), ExitCode.OK) + + def test_rejected_is_payload_error(self): + self.assertEqual(decide_exit(Outcome.REJECTED), ExitCode.REJECTED) + + def test_closed_is_its_own_code(self): + self.assertEqual(decide_exit(Outcome.CLOSED), ExitCode.CLOSED) + + def test_auth_fail_is_its_own_code(self): + self.assertEqual(decide_exit(Outcome.AUTH_FAIL), ExitCode.AUTH) + + def test_unknown_is_undetermined(self): + self.assertEqual(decide_exit(Outcome.UNKNOWN), ExitCode.UNDETERMINED) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +python3 -m unittest tests.test_fire -v +``` + +Expected: `ModuleNotFoundError: No module named 'fastform.fire'` + +- [ ] **Step 3: Write the implementation** + +Create `fastform/fire.py`: + +```python +"""Fetch a form, fill it, submit it once, and report what happened. + +Deliberately single-shot: there is no retry path anywhere in this file. At this +stage nothing has measured whether a second POST overwrites the first, so the +only safe number of sends is one. Spec Phase 6 replaces this with measured +retry discipline. +""" + +import argparse +import subprocess +import sys +import urllib.parse +from enum import IntEnum + +from fastform import compile as compile_mod +from fastform import classify as classify_mod +from fastform import cookies, match, parse +from fastform.transport import WarmConnection + + +class ExitCode(IntEnum): + OK = 0 + REJECTED = 1 + NOT_PARSEABLE = 2 + CLOSED = 3 + AUTH = 4 + UNDETERMINED = 5 + GAPS = 6 + + +_EXIT_BY_OUTCOME = { + classify_mod.Outcome.RECORDED: ExitCode.OK, + classify_mod.Outcome.ALREADY_RESPONDED: ExitCode.OK, + classify_mod.Outcome.REJECTED: ExitCode.REJECTED, + classify_mod.Outcome.CLOSED: ExitCode.CLOSED, + classify_mod.Outcome.AUTH_FAIL: ExitCode.AUTH, + classify_mod.Outcome.UNKNOWN: ExitCode.UNDETERMINED, +} + + +def decide_exit(outcome: "classify_mod.Outcome") -> ExitCode: + return _EXIT_BY_OUTCOME[outcome] + + +def open_in_browser(url: str) -> None: + """Start the browser fallback loading in parallel. Costs nothing if unused.""" + try: + subprocess.Popen( + ["open", url], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + except OSError: + pass + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(prog="fastform.fire") + ap.add_argument("--url", required=True) + ap.add_argument( + "--no-browser", action="store_true", help="skip opening the fallback tab" + ) + ap.add_argument( + "--force", action="store_true", help="submit even if required answers are missing" + ) + args = ap.parse_args(argv) + + if not args.no_browser: + open_in_browser(args.url) + + cookie = cookies.load() + parts = urllib.parse.urlsplit(args.url) + + conn = WarmConnection(parts.netloc) + conn.connect() + try: + path = parts.path + ("?" + parts.query if parts.query else "") + page = conn.get(path, cookie) + + # Auth check BEFORE parsing. A signed-out sign-in-required form still + # serves the full question structure at HTTP 200, so parsing would + # succeed and we would only learn the truth from a wasted POST. + if parse.requires_sign_in(page.body): + print("AUTH FAIL: the form loaded signed out.") + print("config/cookies.txt is stale — recapture it from DevTools.") + return int(ExitCode.AUTH) + + email = parse.signed_in_email(page.body) + + try: + spec = parse.parse(page.body, args.url) + except parse.NotParseable as exc: + print(f"NOT PARSEABLE: {exc}") + if page.status in (301, 302, 303, 307, 308): + # A shortened forms.gle link lands here. Redirects are not + # followed at this phase — say so rather than blaming the form. + print(f"The URL redirected to: {page.location}") + print("Pass the full https://docs.google.com/forms/d/e/.../viewform URL.") + else: + print("Form is closed, already responded to, or deleted.") + print("Use the browser tab.") + return int(ExitCode.NOT_PARSEABLE) + + filled = match.fill(spec, match.load_bank()) + + if filled.blockers: + print(f"BLOCKED: {len(filled.blockers)} question(s) the direct POST cannot fill:") + for question in filled.blockers: + print(f" type={question.type} {question.text}") + print("Use the browser tab.") + return int(ExitCode.GAPS) + + if filled.gaps and not args.force: + print(f"GAPS: {len(filled.gaps)} required question(s) unmatched:") + for question in filled.gaps: + print(f" entry.{question.entry_id} {question.text}") + print("\nAdd them to config/answers.toml, or re-run with --force.") + return int(ExitCode.GAPS) + + try: + blob = compile_mod.build(spec, filled, cookie, email=email) + except compile_mod.MissingToken as exc: + print(f"CANNOT BUILD REQUEST: {exc}") + print("Use the browser tab.") + return int(ExitCode.NOT_PARSEABLE) + + result = conn.send(blob) + outcome = classify_mod.classify(result.status, result.location, result.body) + + print(f"outcome {outcome.value}") + print(f"status {result.status}") + print(f"elapsed {result.elapsed_ms:.1f} ms") + print(f"confidence {filled.confidence:.2f}") + print(f"account {email or '(none detected)'}") + + if outcome is classify_mod.Outcome.UNKNOWN: + print("\nSubmission status is UNDETERMINED. Check the response sheet.") + + return int(decide_exit(outcome)) + finally: + conn.close() + + +if __name__ == "__main__": + sys.exit(main()) +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +python3 -m unittest tests.test_fire -v +``` + +Expected: `Ran 6 tests` … `OK` + +- [ ] **Step 5: Verify end to end against F1** + +F1 has limit-to-1 off, so this is repeatable. If Task 9 left it on, turn it off first. + +```bash +cd /Users/supa/projects/active/fastform +python3 -m fastform.fire --url '' --no-browser +echo "exit=$?" +``` + +Expected: `outcome RECORDED`, an elapsed time in the low hundreds of milliseconds, and `exit=0`. + +Confirm a new row appeared in F1's Responses tab. **This is the real check** — the classifier saying `RECORDED` is not proof on its own. + +- [ ] **Step 6: Verify the gap path** + +```bash +python3 -m fastform.fire --url '' --no-browser +echo "exit=$?" +``` + +Expected: a `GAPS:` listing (F3 has questions your answer bank does not cover, such as `Rate your confidence`) and `exit=6`. Nothing was submitted. + +- [ ] **Step 7: Verify the blocker path** + +```bash +python3 -m fastform.fire --url '' --no-browser +echo "exit=$?" +``` + +Expected: a `BLOCKED:` line naming the file upload, and `exit=6`. Nothing was submitted. + +- [ ] **Step 8: Run the whole suite** + +```bash +python3 -m unittest discover -s tests -v +``` + +Expected: every test passes, `OK`. + +- [ ] **Step 9: Update the README** + +Replace `README.md` with: + +```markdown +# fastform + +Submits a Google Form fast, by POSTing directly to `formResponse` instead of +driving a browser. + +Design: `docs/superpowers/specs/2026-08-05-fastform-design.md` +Plan: `docs/superpowers/plans/2026-08-05-fastform-phases-0-4.md` + +## Setup + +1. Sign into Google in Chrome. +2. Open any Google Form, DevTools → Network → copy the `Cookie:` request + header, and paste it into `config/cookies.txt` (untracked). +3. Put your answers in `config/answers.toml`. + +Verify the cookie works: + +```bash +python3 -c "from fastform import cookies; print(cookies.verify(cookies.load()))" +``` + +Expected: `(True, 'your@email')` + +## Use + +```bash +python3 tools/dryrun.py --url # build the request, send nothing +python3 -m fastform.fire --url # fill and submit, once +``` + +`fire` submits exactly once and never retries. Exit codes: `0` recorded, +`1` rejected, `2` unparseable, `3` closed, `4` auth failed, `5` undetermined, +`6` gaps or blockers (nothing sent). + +## Tests + +```bash +python3 -m unittest discover -s tests -v +``` + +## Status + +Spec Phases 0–4 complete. Calibration (Phase 5), retry discipline (Phase 6), +and the daemon with clipboard intake and the hold state (Phase 7) are not built. +``` + +- [ ] **Step 10: Commit** + +```bash +git add fastform/fire.py tests/test_fire.py README.md +git commit -m "Add single-shot sender CLI" +``` + +--- + +## Done criteria + +All of these must hold before this plan is considered complete: + +1. `python3 -m unittest discover -s tests -v` passes with no failures. +2. `python3 tools/curl_diff.py --curl fixtures/f1-manual-submit.curl.txt --url ` reports `MISSING — none`. +3. `python3 -m fastform.fire --url ` exits `0` and a new row appears in F1's Responses tab. +4. `python3 -m fastform.fire --url ` exits `6` and reports the file upload as a blocker, without submitting. +5. No secrets are tracked. Verify with `git ls-files | grep -E 'cookies\.txt|\.curl\.txt'` — expected: no output. (`git status` would not reveal this; ignored files do not appear as untracked.) +6. F1 is left with **Limit to 1 response OFF** and **Accepting responses ON**, so it stays usable. + +## What this plan does not build + +Carried forward to the next plan, after calibration supplies the numbers: + +- A second test form with limit-to-one permanently ON, needed by E3's overwrite comparison and not by this plan +- Experiments E3–E9 and the `calibrate.py` harness (spec Phase 5) +- `strategy.py`: retry, the timeout probe, stop conditions (spec Phase 6) +- `intake.py`, `tui.py`, `daemon.py`, `ARMED_HOLDING`, the confirm gate (spec Phase 7) +- Rehearsals (spec Phase 8) + +Two things from the spec's ARMING section are also absent here, and `fire.py` +reports them rather than handling them: + +- **URL normalization and short-link resolution.** `forms.gle` links, chat-paste + punctuation, and redirect wrappers are not handled. Pass the full + `docs.google.com/forms/d/e/.../viewform` URL. This belongs with `intake.py` in + Phase 7; `fire.py` prints the redirect target so the failure is legible. +- **The multi-connection warm pool and heartbeat loop.** See the note in Task 10 + — a hand-run tool has no idle period to warm through. diff --git a/docs/superpowers/specs/2026-08-05-fastform-design.md b/docs/superpowers/specs/2026-08-05-fastform-design.md new file mode 100644 index 0000000..1fe0196 --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-fastform-design.md @@ -0,0 +1,594 @@ +# FastForm — Google Form speed-submission system + +**Date:** 2026-08-05 +**Status:** Approved design, not yet implemented + +## Problem + +A competition distributes a Google Form URL through an unknown channel at an +unknown time. Roughly 1,000 people compete; the first three valid submissions +win. Google sign-in is required. The goal is to minimize elapsed time between +the URL reaching the operator and a valid response being recorded, running from +one machine on a residential connection. + +Nothing about the target form is known in advance: not the URL, not the opening +time, not the question set. No practice replica is available. The system must +therefore discover everything at runtime and keep every assumption in config. + +## Scope boundary: knowable vs. unknowable + +The design splits along what can be learned before the event. + +**Knowable now, by experiment against forms we own.** Auth and cookie mechanics; +what success, validation-rejection, closed-form, and already-responded responses +look like; whether a closed-form POST is a no-op; whether a second POST +overwrites the first; `pageHistory` behavior across sections; `fbzx` staleness; +connection warm-up latency and jitter; rate-limit thresholds. None of this +depends on the target form. + +**Unknowable until the URL lands, therefore pure runtime discovery.** Entry IDs, +question text, question types, required flags, section count, whether sign-in is +required, whether limit-to-one is on, whether a file upload exists. Nothing here +is pre-baked. + +## Design decisions and their rationale + +### Clock sync is not the problem; open-detection is + +Google Forms has no native scheduled open. An announced opening time means a +human toggles "accepting responses" (seconds of slop), an Apps Script trigger +fires (minute granularity, 15–45s drift is routine), or the form was already +open and the announced time is just when the link drops. In none of these cases +does millisecond clock accuracy help. The system polls to detect open rather +than firing on a clock. + +### A closed form may not be parseable + +If the organizer opens by toggling, a GET before open returns the "not accepting +responses" page with no `FB_PUBLIC_LOAD_DATA_` — no entry IDs, no structure. The +payload cannot be compiled. This forces a slower path (poll GET until it renders, +then arm, then POST — roughly two extra round trips) and no part of the design +can avoid it. Acknowledged, not solved. + +### No browser in the hot path, but cookies must be read live + +Authentication needs cookies, not a browser. The fire path is one stdlib HTTPS +POST with a Cookie header. This removes the largest dependency and eliminates a +hard failure mode: `launch_persistent_context` fails outright if Chrome is +already running on the target profile, which it will be on the day. + +**Cookies cannot be captured once and reused.** This was measured, not assumed: +a Cookie header copied from DevTools authenticated for about seven minutes and +was dead across all of Google within twenty. Chrome continuously rotates +`__Secure-1PSIDTS` — in the profile under test it had been refreshed four +minutes earlier, while `__Secure-1PSID` was thirty-five hours old — and once it +rotates, the captured value stops working. The browser session stays perfectly +healthy; only the snapshot dies. + +So the tool reads Chrome's cookie store at run time rather than replaying a +saved header. On macOS that is: the encryption secret from the login Keychain +(`security find-generic-password -s "Chrome Safe Storage"`), a key derived with +PBKDF2-HMAC-SHA1 over `saltysalt` at 1003 iterations, and AES-128-CBC over each +`v10`-prefixed value with a sixteen-space IV. Verified end to end with zero +Python dependencies, using an `openssl enc` subprocess for the cipher. + +Two consequences worth stating plainly. A boot-time cookie check hours before +the event proves nothing about the cookie that will exist at fire time. And +profile choice is a real decision, not a detail — the machine under test had two +signed-in Chrome profiles on different Google accounts. + +### Connection warm-up is the largest deterministic win + +Cold DNS + TCP + TLS to `docs.google.com` on residential is 150–350ms with high +variance — likely larger and jitterier than everything else combined. The system +holds 2–3 warm TLS connections with a keep-alive heartbeat and fires on an +already-handshaken socket. Multiple sockets give microsecond failover instead of +a fresh handshake when a send raises. + +### Classify by structure, not by string + +Google renders these pages in the account's UI language, so string markers are +fragile. The structural signals below were measured against real responses, and +they replace an earlier guess that was wrong in a dangerous direction: + +| Signal | Meaning | +|---|---| +| `data-sign-in-to-continue="true"` | Signed out. Google serves this as HTTP 200 with the full form, never a redirect. | +| Response still contains ``. The +tab loads in parallel with parsing, costs one line and no dependency, and +carries no duplicate risk because the tab does nothing until touched. The +operator arbitrates between the automated path and the rendered tab. + +### Stage the fallback, do not race it + +The POST resolves in 200–400ms; a manual browser fill takes 2–5s. Firing the +browser 400ms later costs about 10% of the fallback's own latency and removes +duplicate risk entirely. Sequential with an aggressive timeout, not parallel. + +## Architecture + +Single long-running Python process (a reactive daemon) built over a two-script +seam: parse/compile and transport remain independently runnable, so if the +daemon misbehaves the operator runs two commands by hand and loses automation, +not capability. + +``` +fastform/ + config/ + answers.toml # answer bank: text patterns -> values + profile.toml # timeouts, poll intervals, gate threshold, hold policy + classify.toml # response markers (emitted by calibration, not hand-written) + cookies.txt # extracted Google auth cookies (gitignored) + fastform/ + cookies.py # Chrome profile -> cookie header; freshness check + parse.py # URL or HTML bytes -> FormSpec + match.py # FormSpec + answer bank -> FilledForm (+ gaps, confidence) + compile.py # FilledForm -> raw HTTP request bytes + transport.py # warm connection pool, keep-alive, send, timing + classify.py # response bytes -> Outcome + strategy.py # the fire loop and its stop conditions + intake.py # clipboard / stdin / LAN -> one URL queue + tui.py # state display + confirm gate + daemon.py # state machine, wires the above + tools/ + calibrate.py # experiments against our own forms -> classify.toml + dryrun.py # build request, print it, do not send +``` + +### Data types + +- **`FormSpec`** — form id, action URL, page count, sign-in required, + limit-to-one, and `Question(entry_id, text, type, required, options, + validation)` list. Produced only by `parse.py`. Pure data, no network. +- **`FilledForm`** — `entry.N=value` pairs, `gaps` (required questions nothing + matched), `blockers` (file uploads, unsupported types), confidence score. +- **`RequestBlob`** — literal bytes for the socket, plus host and port. Nothing + downstream of this knows what a form is. +- **`Outcome`** — `RECORDED | REJECTED | CLOSED | ALREADY_RESPONDED | + AUTH_FAIL | UNKNOWN`. `classify.py` is a pure function from response bytes to + this enum, so it tests trivially against saved fixtures. It classifies both + POST responses and probe-GET bodies — `ALREADY_RESPONDED` in particular is + normally only observable on a GET, not in the POST response itself. + +### Module boundaries + +`parse` makes no policy decisions. `transport` does not know what a form is. +`classify` is pure. `strategy` is the only module that decides whether to send +again — every "will this double-submit me" question lives in one readable file. + +### Dependencies + +Hot path is stdlib only: `http.client`, `ssl`, `socket`, `json`, `re`, +`difflib`, `threading`, `http.server`, `tomllib`. No web framework, no database, +no ORM. Config is three TOML files plus a cookie file. + +The one place stdlib fights us is the clipboard: `pbpaste` forks a subprocess per +poll, so polling it at 10ms is wasteful and jittery. Primary approach is a ~30 +line `ctypes` binding to `NSPasteboard.changeCount` (an integer read, effectively +free at 5ms polling), keeping the project at zero dependencies. If that binding +proves troublesome during Phase 7, the fallback order is +`pyobjc-framework-Cocoa` as a single dependency, then `pbpaste` at 25ms as a +degraded no-dependency mode. + +## Runtime behavior + +### State machine + +`WAITING → ARMING → ARMED → FIRING → SETTLED`, with connection warm-up running +as a background service throughout. + +Two sub-states: + +- **`PRE_OPEN`** (under `ARMING`) — the URL resolved to a form that is not yet + accepting responses, so there is no structure to parse. See ARMING below. +- **`ARMED_HOLDING`** (under `ARMED`) — the payload is built but `open_at` has + not yet arrived. See Holding below. + +**Boot** happens hours early and is where failures belong: load config, load +cookies, then perform a real authenticated GET and confirm the account email +comes back. Dead cookies found at T-2h are an inconvenience; found at T+0 they +are the loss. + +**Warm pool** maintains 2–3 open TLS connections to `docs.google.com` and to +`forms.gle` (the URL may arrive shortened), heartbeating every 5–10s. + +### Intake + +Three channels feed one queue in `intake.py`, all optional and independently +toggleable, because the delivery channel is unknown: + +- **stdin** — the operator types or pastes into the waiting prompt. Covers the + "URL read out verbally" case. Always available; the only channel present in + the Phase 1–4 version. +- **clipboard** — `NSPasteboard.changeCount` polled at 5ms, so copying the URL + is the operator's only action. Any URL-shaped string on the clipboard arms the + system. +- **LAN listener** — `http.server` in a thread with a single endpoint, paired + with an iOS share-sheet Shortcut, for the case where the URL lands on a phone. + +Whichever channel delivers first triggers arming. Intake does **not** shut down +at that point: during `ARMED_HOLDING` a later URL that differs from the held one +supersedes it and re-arms, because an organizer replacing the form with a new +link is a real possibility and the held payload would be worthless. A repeat of +the same URL is ignored. Intake stops only on entering `FIRING`. + +All three channels normalize through the same path described below. + +### ARMING + +First action on URL receipt, before anything else: `open `, fire and forget. + +Then: normalize the URL (strip trailing punctuation from a chat paste, unwrap +redirect wrappers, resolve `forms.gle` on the warm socket, handle both +`/d/e//viewform` and the rarer `/d//` published path, drop `?usp=`), GET +over a warm socket, parse, match, compile. + +The GET has four outcomes: + +| Response | Action | +|---|---| +| Form renders | Parse, match, compile. Full speed available. | +| Closed page | No entry IDs, cannot compile. Enter `PRE_OPEN`. | +| Sign-in redirect | Cookies dead. Stop loud, hand to the browser tab. | +| 404 | Bad URL. Return to `WAITING`, keep listening. | + +**`PRE_OPEN`** GET-polls at `poll_ms` until the form renders, then completes +arming (parse, match, compile) and proceeds to `FIRING`. Because the form is +known open on exit from `PRE_OPEN`, firing always uses `single` — `optimistic` +has nothing left to save. `PRE_OPEN` is the slow path, costing roughly two extra +round trips versus arriving at an already-open form, and that cost is +structural. + +On successful arming the system enters `ARMED_HOLDING` if `open_at` is set and +still in the future, and `FIRING` otherwise. + +### Holding + +If `open_at` is set in `profile.toml` and has not yet passed, arming completes +into `ARMED_HOLDING` rather than firing. This is the "URL leaked early" case, +and holding is the default because firing early is a one-way door: a submission +recorded two hours before the announced open may be discarded by the organizer, +and under limit-to-one there is no second attempt. If `open_at` is unset, +arming proceeds directly to `FIRING`. + +The point of holding is not the wait — it is that **parsing, matching, and gap +reporting all happen immediately on URL receipt**, hours before they matter. An +unmatched required question discovered at 1pm is an answer-bank edit; the same +question discovered at 3:30:00 is the loss. The gate renders as soon as arming +completes and stays on screen for the duration of the hold, showing the filled +payload, any gaps, and a countdown. + +During the hold: + +- **Gaps stay editable.** Fill them at any point; the payload recompiles. +- **The form is re-validated** on an interval. An organizer may edit the form, + close it to reset, or replace it between leak and open. If the question set + changes, entry IDs may go stale, so a re-GET that detects a structural change + re-runs matching and flags it loudly rather than firing a payload built + against a form that no longer exists. +- **A replacement URL supersedes the held one.** Intake stays live throughout + the hold; a different URL re-arms from scratch. +- **A keypress fires early**, overriding the hold, for the case where the + operator judges that the announced time is not binding. + +At `open_at`, firing proceeds automatically using whichever mode applies. + +### Matching + +Confidence ladder, highest first: + +1. Explicit regex from `answers.toml` matches question text → 1.0 +2. Exact normalized equality (casefolded, depunctuated, whitespace-collapsed) → 0.95 +3. All pattern tokens present in the question text → 0.8 +4. `difflib.SequenceMatcher` ratio above threshold → the ratio itself +5. No match → gap + +For choice-type questions the *value* must also match an entry in the option +list. An answer of "Bangkok" against an option of "Bangkok Metropolitan Region" +is a silent-rejection generator, so a value matching no option counts as a gap +even when the question matched perfectly. + +Auto-fire requires `confidence >= threshold AND no gaps AND no blockers`. + +Gaps go to the gate. **Blockers bypass the gate entirely** and go straight to +browser handoff — a file upload cannot be resolved with a keypress at a +terminal, and presenting it at the gate would waste the seconds that matter. + +### The gate + +Full-screen terminal render: every question, matched value, confidence, gaps in +red. Choice gaps get numbered options so one keypress fills them; free-text is +typed. Bare `Enter` fires; `b` abandons to the already-loaded browser tab. Raw +tty so single keypresses register without Enter. Renders in under 50ms, +pre-rendered as far as possible during ARMING. + +Policy on unknown required questions is config (`unknown_required` = +`prompt` | `first_valid_option` | `abort`), defaulting to `prompt`. Required free +text is never auto-guessed under any policy — `first_valid_option` applies only +to choice-type questions, and falls back to `prompt` for free text. + +Unmatched **non-required** questions are omitted from the payload entirely rather +than filled with a guess. They cannot cause a rejection, and omitting them keeps +the payload minimal. + +### FIRING + +Two modes in `strategy.py`, selected at runtime from what `parse.py` learned +rather than guessed in config: + +- **`single`** — one POST, classify, done. Used whenever the form is known open + at the moment of firing: no hold was configured, or the hold expired with the + form still accepting. Always used on exit from `PRE_OPEN`, since the form is + known open there by definition. +- **`optimistic`** — POST every `poll_ms` until terminal. Reachable in exactly + one situation: we hold a parsed `FormSpec` from `ARMED_HOLDING`, and hold-time + re-validation shows the form has since closed — the organizer shut it to reset + before the announced open. Because entry IDs are already in hand, POST-polling + through the reopen saves the full round trip that `PRE_OPEN` cannot avoid. + + **Now high-risk, and off by default.** It was gated on a closed-form POST + being a no-op, which holds — a closed form answers 302 to `/closedform` and + records nothing. But it also implicitly assumed some server-side duplicate + protection, and measurement found none: every accepted POST appends a row. A + single missed stop, or one ambiguous timeout mid-poll, writes duplicate + entries. Enable only with `max_sends` of 2 or 3 and only after rehearsing the + stop path; otherwise this case degrades to `PRE_OPEN` and costs one round trip. + + A further catch: `token` is per-page-load, so a POST-poll cannot reuse one + payload indefinitely. Token lifetime is unmeasured — treat it as a calibration + item before relying on this mode. + +Stop conditions: + +| Outcome | Action | +|---|---| +| `RECORDED` | Hard stop. No further sends under any circumstance. | +| `ALREADY_RESPONDED` | Stop. Treated as success. Measured absent on this form — retained only in case a differently configured form produces it. | +| `REJECTED` | Stop. Payload is wrong; repeating will not help. Surface to gate. | +| `CLOSED` | Keep polling — expected pre-open state. | +| `AUTH_FAIL` | Stop loud, hand to browser. | +| `UNKNOWN` | Stop sending. Probe once; if inconclusive, hand to browser with a loud warning that submission status is undetermined. Never resend on an unclassifiable response. | +| timeout / socket error | Probe, never blind-retry. Behavior depends on limit-to-one, below. | + +Plus a hard `max_sends` cap and a wall-clock deadline, both config. + +**Timeout behaviour: resend, because nothing else is available.** This replaces +an earlier design that was built on a mechanism which turns out not to exist. + +Three protections were assumed and all three were measured absent: + +| Assumed protection | Measured reality | +|---|---| +| `fbzx` deduplicates repeats | No. Three identical POSTs sharing one `fbzx`, `token` and `tag` produced three separate rows. | +| "Limit to 1 response" blocks a second entry | No. With the setting on and prior responses from the same account, a direct POST appended a new row and changed nothing existing. | +| `ALREADY_RESPONDED` is observable | No such state on either verb. The GET returns an ordinary form; the POST returns an ordinary confirmation. | + +So there is **no server-side idempotency of any kind**, and no signal that +distinguishes "your send landed" from "your send was lost". The check-then-act +policy this spec originally specified has nothing to check. + +Given that, on a timeout the choice is between a possible duplicate row and a +possible missing submission. Resend, up to `max_sends`. In a first-three-wins +race the organizer takes the earliest qualifying entry, so a duplicate costs +approximately nothing while a lost submission costs everything. + +Two consequences worth stating plainly: + +- **The client is the only safeguard.** `RECORDED` must hard-stop the loop, and + that stop is enforced by nothing but our own code. A bug there produces real + duplicate entries. +- **Duplicates are visible to the organizer.** Several identical rows under one + account may read as abuse even where the rules permit automation. Keep + `max_sends` low — 2 or 3, not 10. + +`UNKNOWN` remains more conservative than a timeout: a timeout means the send may +not have landed, while an unclassifiable response means it may well have. Stop +and surface it rather than resending blind. + +### Jitter control + +- `gc.freeze()` on entering ARMED, moving everything allocated so far into the + permanent generation. `gc.disable()` only across the actual send window, not + from ARMED to SETTLED — in `PRE_OPEN` or a long `optimistic` run that window is + minutes long and allocates a response body per poll, so the collector stays on + between polls and is disabled only around each send. +- Blob fully materialized at arm time; the send path allocates nothing. +- Fire on the main thread. **Intake threads stop on entering `FIRING`**, not on + arming — a replacement URL during `ARMED_HOLDING` must still be able to land. + Stopping them matters because the 5ms clipboard poll is a spin loop, not a + blocked read, and would otherwise contend during exactly the window being + protected. +- Stage timestamps via `perf_counter_ns()` into a preallocated list, flushed + after SETTLED. No logging I/O in the hot path. +- Run under `caffeinate -dimsu` in a foreground terminal to avoid App Nap and + timer coalescing. +- Ethernet where available. Wi-Fi power-save adds 100ms+ to the first packet + after idle and cannot be fixed in software. + +Every failure path terminates in either a definitive `Outcome` or the +already-loaded browser tab. Nothing degrades silently. + +## Testing and calibration + +### Test bed (Phase 0, operator-created) + +Four forms in our own Google account, chosen to cover branches rather than to be +realistic: + +- **F1** — one section, sign-in required, **limit-to-1 ON**, three required + short answers. Auth and overwrite testbed. +- **F2** — identical but **limit-to-1 OFF**. Duplicate semantics; load-test + target, since extra rows are harmless. +- **F3** — three sections, mixed types (short answer, paragraph, multiple choice + with "Other", checkboxes, dropdown, linear scale, date, time), mixed required + flags, one section branch. `pageHistory` testbed. +- **F4** — regex validation on a short answer, a number range, email format, + length limits, plus one **file upload** question. Rejection-path and + blocker-detection testbed. + +### Experiments + +Each emits fixtures and populates config. + +- **E1 — cookie-only POST works?** Submit to F1 with nothing but a Cookie + header; confirm the row lands in the responses sheet under the account. This + is the go/no-go for the entire approach. On failure, diff against a DevTools + "copy as cURL" to find what is missing. +- **E2 — response fingerprints.** Capture full response bytes for success, + validation-rejected, closed, already-responded, and dead-cookies. These become + `classify.py`'s unit tests. Record status codes, redirects, and whether + `FB_PUBLIC_LOAD_DATA_` reappears. Note locale strings without depending on them. +- **E3 — overwrite semantics.** ANSWERED. Three identical POSTs sharing one + `fbzx`/`token`/`tag` produced three rows; with "Limit to 1 response" on, a + direct POST still appended a row and altered nothing existing. There is no + server-side deduplication and no overwrite. Re-run only to check whether this + changes. +- **E4 — is a closed-form POST a no-op?** Toggle a form closed, POST, reopen, + verify nothing from the closed window appears. Optimistic polling is unsafe + without this. +- **E5 — `pageHistory` on F3.** Does one POST with `0,1,2` satisfy all sections? + What happens when it is wrong or short? Does the branch change what is accepted? +- **E6 — `fbzx` staleness.** Arm from a page load, wait two hours, fire — still + accepted? Does a self-generated random `fbzx` work, or must it come from the + page? Determines whether arming hours early is a real capability. +- **E7 — latency and jitter profile.** Fifty submissions to F2, cold socket vs. + warm, recording p50/p95/max. Sets the send timeout (from p95, not p50 — the + timeout exists to catch the tail) and quantifies what warm-up actually buys. + `max_sends` is a safety cap, not a latency figure, and is set from E3's + overwrite findings instead. +- **E8 — rate limiting.** Poll GET at the intended interval for 60s. Any 429 or + captcha? Sets a safe `poll_ms`. +- **E9 — warm connection survival.** Hold a TLS connection to `docs.google.com` + with the intended heartbeat for three hours, then fire on it and confirm the + submission lands. Google may reap idle connections, NAT mappings expire, and + consumer routers drop state. The entire warm-pool premise rests on this and + nothing else tests it. If connections do not survive, the fix is a shorter + heartbeat or periodic reconnection — determine which, and set the heartbeat + interval from the measured survival time rather than the assumed 5–10s. + +### Highest-value single test + +Manually submit F3 in Chrome with DevTools open, "copy as cURL", then run +`tools/dryrun.py` against the same form and diff the two requests field by +field. This catches every missing token at once. It must be a repeatable script, +re-run after every parser change. + +### Unit tests + +All pure-function-against-fixture: `parse.py` on saved HTML, `classify.py` on +saved responses, `match.py` on synthetic `FormSpec`s built to be nasty +(near-duplicate question text, a value matching no option, an unmatched required +question). + +### Rehearsal + +End-to-end against F3 with a stopwatch, both scenarios (URL arrives early, URL +arrives at open), repeated until the gate keystrokes are muscle memory. At least +one rehearsal with something deliberately broken, so the degraded path has been +practiced rather than met for the first time on the day. + +## Build order + +| Phase | Contents | Runnable result | +|---|---|---| +| 0 | Create F1–F4, capture a manual cURL | — | +| 1 | `cookies.py` + a 20-line hardcoded POST to F1 | **E1 go/no-go gate** | +| 2 | `parse.py` + fixtures + tests | `parse ` prints FormSpec | +| 3 | `match.py`, `answers.toml`, `compile.py`, `dryrun.py` | URL → request bytes; run the cURL diff | +| 4 | `classify.py` + E2 fixtures, `transport.py` warm pool, **single-shot sender** | Two-script seam complete, usable by hand | +| 5 | Calibration harness: E3–E9 | emits `classify.toml`, measured `profile.toml` | +| 6 | `strategy.py` with the stop-condition table | driven by Phase 5 findings | +| 7 | `intake.py` (stdin → clipboard → LAN), `tui.py`, `daemon.py` incl. `ARMED_HOLDING` | the daemon | +| 8 | Rehearsals | — | + +**Phase 1 is a hard gate.** If cookie-only POST does not work the design changes +fundamentally and everything after Phase 1 is wasted. Build nothing past it +until E1 passes. + +**Phase 4's sender fires exactly once.** One POST, classify, print the outcome, +stop. It contains no loop and no resend path, so it cannot double-submit — the +safety is structural, not logical, which matters because at Phase 4 no +calibration has run and there is nothing on which to base correct retry +behavior. `strategy.py` in Phase 6 replaces it with measured retry discipline. +Until then, a timed-out send is treated as a loss and handed to the browser. + +**Holding is manual before Phase 7.** `ARMED_HOLDING` lives in `daemon.py`, so +in the Phase 1–4 version the operator holds by running the parse step early and +the send step at the chosen moment. The two-script seam makes this natural, and +it preserves the property that matters most — gaps surface at parse time, hours +before firing. + +**Phases 1–4 are the first usable version.** Paste a URL in a shell, get a +submission in roughly one to two seconds including typing. It is competitive on +its own, and it is what to fall back to if later phases are incomplete when the +event arrives. What Phases 5–8 add is speed and the ability to *recover*: +without `strategy.py`, a timed-out send is simply a loss, because nothing is +allowed to investigate or resend. That is an acceptable trade for a version with +no measurements behind it, but it is a real limitation, not a rough edge. + +The event date is unknown, so there is no deadline to plan against. Build in +phase order and stop wherever you are; every phase boundary is a usable state. + +## Out of scope + +Deliberately not built: + +- Playwright or any browser automation. The browser fallback is a tab the + operator drives. +- Millisecond clock synchronization. The opening event is not clock-driven. +- A Chrome extension. +- Headless-browser fallback. +- Retry-with-backoff in any form. Retry is check-then-act or nothing. + +Relative to the Phase 1–4 version, these are gold-plating and may be dropped if +time runs short: the LAN listener (stdin works), the ctypes clipboard binding +(stdin works), the multi-socket warm pool (one socket works), optimistic polling +(single-shot works), the TUI (a printed table and `input()` works). + +## Known risks + +- **Cookie-only POST may not be accepted.** Mitigated by making it Phase 1 and a + hard gate rather than an assumption. +- **The form may be closed and unparseable when the URL arrives**, forcing the + slow path. Structural, unfixable, and affects every competitor equally. +- **The answer bank may not match the real question text**, producing gaps. The + gate exists precisely for this and must be rehearsed. +- **A file upload question breaks the direct-POST path entirely.** Detected at + parse time as a blocker and routed to the browser. Detection is cheap; solving + it is not, and would lose the race anyway. +- **Google cookies may hit a "verify it's you" challenge** on the day. The + boot-time authenticated GET only partly mitigates this. It detects a signed-out + session by the redirect to `accounts.google.com`, but a captcha, consent, or + "unusual traffic" interstitial served with HTTP 200 is indistinguishable from a + signed-in page by status alone, and would be reported as authenticated. Closing + that gap needs a captured sample of such a page to derive a marker from, which + calibration should collect if one ever appears. Until then, treat a green + boot-time check as necessary but not sufficient, and confirm by eye that a form + actually renders. +- **The form may be edited between an early leak and the announced open**, + staling the entry IDs a held payload was built against. Mitigated by hold-time + re-validation, which re-runs matching on a detected structural change. A + wholly different replacement URL is handled by intake staying live through the + hold, but only if that new URL actually reaches the operator. +- **Organizer rules may forbid automated submission.** A disqualification risk + independent of any technical decision here; the operator has accepted it. diff --git a/fastform/__init__.py b/fastform/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fastform/classify.py b/fastform/classify.py new file mode 100644 index 0000000..ca9b40b --- /dev/null +++ b/fastform/classify.py @@ -0,0 +1,86 @@ +"""Decide what a response means, from measured fixtures — not guesses. + +Every rule here was derived from real captured responses (see fixtures/README). +The signals are structural and locale-independent, which matters because the +account under test renders Forms in Thai. An earlier version keyed on +`FB_PUBLIC_LOAD_DATA_` reappearing and on `freebird*` CSS classes; both were +wrong — the success page also carries `FB_PUBLIC_LOAD_DATA_`, and those classes +do not exist in current markup. That mistake reported a recorded submission as +REJECTED, which is the worst error available because it invites a retry. +""" + +from enum import Enum +from urllib.parse import urlsplit + +#: The actual element. Present only when the form came back for +#: another attempt, i.e. the submission did NOT record. This is the primary +#: recorded-vs-rejected discriminator. +FORM_ELEMENT = b" bool: + return any(marker in body for marker in markers) + + +def classify(status: int, location: str | None, body: bytes) -> Outcome: + # Redirects carry the signal before any body is read. A closed form 302s to + # /closedform; a dead session 302s to accounts.google.com. + if status in (301, 302, 303, 307, 308): + loc = location or "" + if "accounts.google.com" in loc: + return Outcome.AUTH_FAIL + if urlsplit(loc).path.endswith("/closedform"): + return Outcome.CLOSED + return Outcome.UNKNOWN + + if not body: + return Outcome.UNKNOWN + + # Signed-out sign-in-required form: HTTP 200 with the whole form present. + # Must precede the form-element test below. + if SIGN_IN_MARKER in body: + return Outcome.AUTH_FAIL + + # The closed landing page, if a redirect was followed through to it. + if CLOSED_MARKER in body and LOAD_DATA not in body: + return Outcome.CLOSED + + # A re-rendered form means the submission did not take. Measured rejects come + # back 400, but keying on the form element as well as the status is stronger. + if status == 400 or FORM_ELEMENT in body: + return Outcome.REJECTED + + # Terminal confirmation page: no form element, positive recorded marker. + if _has_any(body, RECORDED_MARKERS): + return Outcome.RECORDED + + return Outcome.UNKNOWN diff --git a/fastform/compile.py b/fastform/compile.py new file mode 100644 index 0000000..c21f5f0 --- /dev/null +++ b/fastform/compile.py @@ -0,0 +1,85 @@ +"""Turn a filled form into a ready-to-send request. + +Everything expensive happens here, at arm time. Sending is then a socket write +with nothing left to compute — which is the whole point of separating this from +transport. +""" + +import random +import urllib.parse +from dataclasses import dataclass + +from fastform.cookies import USER_AGENT +from fastform.match import FilledForm +from fastform.parse import FormSpec + +HOST = "docs.google.com" + +#: Hidden inputs the server rejects the POST without. Established by dropping +#: each field in turn against a real form: omitting either yields HTTP 400. +REQUIRED_HIDDEN = ("token", "tag") + + +class MissingToken(Exception): + """The page did not yield `token`/`tag`, so no POST can succeed. + + Raised rather than sending a request that is certain to 400 — the failure is + better surfaced at arm time than as a mystery rejection mid-run. + """ + + +@dataclass +class RequestBlob: + host: str + path: str + body: str + headers: dict[str, str] + + +def new_fbzx() -> str: + """A fresh anti-duplicate token, in the same shape Google's own pages emit.""" + return str(random.randint(-(2**63), 2**63 - 1)) + + +def build( + spec: FormSpec, + filled: FilledForm, + cookie: str, + email: str | None = None, + fbzx: str | None = None, +) -> RequestBlob: + missing = [k for k in REQUIRED_HIDDEN if not spec.hidden.get(k)] + if missing: + raise MissingToken( + f"page yielded no {', '.join(missing)} — the POST would be rejected" + ) + + fields: list[tuple[str, str]] = [ + (f"entry.{entry_id}", value) for entry_id, value in filled.pairs + ] + if email: + fields.append(("emailAddress", email)) + fields.append(("fvv", "1")) + # Prefer the page's own fbzx; it is already paired with this token. + fields.append(("fbzx", fbzx or spec.hidden.get("fbzx") or new_fbzx())) + fields.append(("pageHistory", spec.page_history)) + for key in REQUIRED_HIDDEN: + fields.append((key, spec.hidden[key])) + fields.append(("submissionTimestamp", "-1")) + + body = urllib.parse.urlencode(fields, encoding="utf-8") + viewform = f"https://{HOST}/forms/d/e/{spec.form_id}/viewform" + + return RequestBlob( + host=HOST, + path=spec.action_path, + body=body, + headers={ + "Cookie": cookie, + "User-Agent": USER_AGENT, + "Content-Type": "application/x-www-form-urlencoded", + "Content-Length": str(len(body.encode("utf-8"))), + "Referer": viewform, + "Origin": f"https://{HOST}", + }, + ) diff --git a/fastform/cookies.py b/fastform/cookies.py new file mode 100644 index 0000000..bba32f1 --- /dev/null +++ b/fastform/cookies.py @@ -0,0 +1,237 @@ +"""Google auth cookies: read them live from Chrome, or load a manual override. + +A Cookie header captured by hand dies within minutes — Chrome rotates +`__Secure-1PSIDTS` continuously, and the snapshot stops authenticating once it +does (measured: dead across all of Google within twenty minutes). So the real +source is `cookie_header_from_chrome()`, which decrypts Chrome's live store at +call time. `load()` remains only as a manual override for a hand-pasted header. +""" + +import hashlib +import http.client +import re +import shutil +import sqlite3 +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path + +USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36" +) + +DEFAULT_PATH = Path(__file__).resolve().parent.parent / "config" / "cookies.txt" + +_EMAIL_RE = re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+") + + +def load(path: Path | None = None) -> str: + """Return the bare Cookie header value, with any 'Cookie:' prefix removed.""" + path = path or DEFAULT_PATH + raw = path.read_text(encoding="utf-8").strip() + if raw.lower().startswith("cookie:"): + raw = raw.split(":", 1)[1].strip() + if not raw: + raise ValueError(f"{path} is empty — paste a Cookie header into it") + return raw + + +def verify(cookie_header: str) -> tuple[bool, str | None]: + """GET a signed-in-only page. Returns (signed_in, email_if_found). + + Signed-out sessions get redirected to accounts.google.com, which is a + structural signal and does not depend on the account's UI language. + + Limitation: any HTTP 200 response is treated as signed in. A captcha or + "unusual traffic" interstitial served with status 200 would be + misread as an authenticated session -- this function does not attempt + to detect that case. + """ + conn = http.client.HTTPSConnection("docs.google.com", timeout=15) + try: + conn.request( + "GET", + "/forms/u/0/", + headers={"Cookie": cookie_header, "User-Agent": USER_AGENT}, + ) + resp = conn.getresponse() + location = resp.getheader("Location") or "" + body = resp.read() + finally: + conn.close() + + if resp.status in (301, 302, 303, 307, 308) and "accounts.google.com" in location: + return False, None + if resp.status != 200: + return False, None + + match = _EMAIL_RE.search(body.decode("utf-8", "replace")) + return True, match.group(0) if match else None + + +#: Chrome's macOS cookie encryption, all constants fixed by Chromium itself. +_SALT = b"saltysalt" +_ITERATIONS = 1003 +_KEY_LENGTH = 16 +_IV = b" " * 16 +_ENCRYPTED_PREFIXES = (b"v10", b"v11") + +#: Only these host_keys are sent to docs.google.com. Filtering here matters: +#: each value costs one openssl fork, and this cuts ~91 cookies down to ~24. +_GOOGLE_HOSTS = (".google.com", "docs.google.com", ".docs.google.com") + +CHROME_ROOT = Path.home() / "Library/Application Support/Google/Chrome" + + +class ChromeCookieError(Exception): + """Chrome's cookie store could not be read or decrypted.""" + + +@dataclass +class ChromeProfile: + name: str + path: Path + google_cookies: int + + +def keychain_secret() -> str: + """Chrome's encryption password, stored in the login Keychain. + + macOS may prompt for permission the first time. That prompt is the reason + this is worth doing well before the event rather than during it. + """ + proc = subprocess.run( + ["security", "find-generic-password", "-w", "-s", "Chrome Safe Storage"], + capture_output=True, text=True, timeout=30, + ) + if proc.returncode != 0: + raise ChromeCookieError( + f"could not read Chrome Safe Storage from Keychain: {proc.stderr.strip()}" + ) + return proc.stdout.strip() + + +def derive_key(secret: str) -> bytes: + return hashlib.pbkdf2_hmac( + "sha1", secret.encode(), _SALT, _ITERATIONS, _KEY_LENGTH + ) + + +def decrypt_value(blob: bytes, key: bytes) -> str | None: + """Decrypt one cookie value. Returns None for anything not v10/v11. + + Python has no stdlib AES, so this shells out to openssl rather than taking + a dependency. -nopad is required because we strip PKCS7 ourselves; letting + openssl do it makes a wrong key raise instead of returning garbage. + """ + if not blob or blob[:3] not in _ENCRYPTED_PREFIXES: + return None + proc = subprocess.run( + ["openssl", "enc", "-d", "-aes-128-cbc", + "-K", key.hex(), "-iv", _IV.hex(), "-nopad"], + input=blob[3:], capture_output=True, + ) + if proc.returncode != 0 or not proc.stdout: + return None + out = proc.stdout + pad = out[-1] + if 1 <= pad <= 16: + out = out[:-pad] + # Chrome >= v24 prefixes 32 bytes of domain hash before the value. + if len(out) > 32 and not re.fullmatch(rb"[\x20-\x7e]*", out[:32]): + out = out[32:] + return out.decode("utf-8", "replace") + + +def _read_cookies(db_path: Path) -> list[tuple[str, bytes]]: + """Copy first — Chrome holds a lock on the live database.""" + with tempfile.TemporaryDirectory() as tmp: + copy = Path(tmp) / "Cookies" + shutil.copy(db_path, copy) + con = sqlite3.connect(copy) + try: + placeholders = ",".join("?" * len(_GOOGLE_HOSTS)) + return con.execute( + f"SELECT name, encrypted_value FROM cookies " + f"WHERE host_key IN ({placeholders})", _GOOGLE_HOSTS + ).fetchall() + finally: + con.close() + + +def list_profiles(root: Path | None = None) -> list[ChromeProfile]: + """Every Chrome profile holding Google cookies, most cookies first. + + Profiles matter: a machine can have several signed into different accounts, + and the wrong one submits as the wrong person. + """ + root = root or CHROME_ROOT + if not root.is_dir(): + return [] + found = [] + for child in sorted(root.iterdir()): + db = child / "Cookies" + if not db.is_file(): + continue + try: + count = len(_read_cookies(db)) + except (sqlite3.Error, OSError): + continue + if count: + found.append(ChromeProfile(child.name, child, count)) + return sorted(found, key=lambda p: -p.google_cookies) + + +def _decrypt_profile(chosen: ChromeProfile) -> str: + key = derive_key(keychain_secret()) + jar: dict[str, str] = {} + for name, blob in _read_cookies(chosen.path / "Cookies"): + value = decrypt_value(blob, key) + if value is not None: + jar.setdefault(name, value) + if not jar: + raise ChromeCookieError(f"decrypted no cookies from profile {chosen.name!r}") + return "; ".join(f"{k}={v}" for k, v in jar.items()) + + +def cookie_header_from_chrome(profile: str) -> str: + """Build a Cookie header from a named Chrome profile's live store. + + `profile` is required, not optional. A machine can have several Chrome + profiles signed into different Google accounts, and cookie count — the only + thing distinguishing them without a network call — is not a basis for + choosing which identity submits. Pick with `resolve_accounts()` once, pin + the name in config, and pass it here. + + Read at call time, never cached: Chrome rotates __Secure-1PSIDTS + continuously, and a header captured twenty minutes ago no longer + authenticates anywhere on Google. + """ + profiles = list_profiles() + if not profiles: + raise ChromeCookieError(f"no Chrome profile with Google cookies under {CHROME_ROOT}") + chosen = next((p for p in profiles if p.name == profile), None) + if chosen is None: + names = ", ".join(p.name for p in profiles) + raise ChromeCookieError(f"no profile named {profile!r}; found: {names}") + return _decrypt_profile(chosen) + + +def resolve_accounts(root: Path | None = None) -> list[tuple[ChromeProfile, str | None]]: + """Pair each profile with the Google account it is signed into. + + A setup-time helper, not a hot-path call: it makes one authenticated GET per + profile. Cookie count tells the operator nothing about *which* account a + profile holds; this turns "Profile 6" into an email they can recognise. A + profile whose cookies no longer authenticate resolves to None. + """ + resolved = [] + for prof in list_profiles(root): + try: + _, email = verify(_decrypt_profile(prof)) + except ChromeCookieError: + email = None + resolved.append((prof, email)) + return resolved diff --git a/fastform/match.py b/fastform/match.py new file mode 100644 index 0000000..5da411b --- /dev/null +++ b/fastform/match.py @@ -0,0 +1,109 @@ +"""Match prepared answers to a form's questions by text, never by position. + +Position matching breaks the moment a question is added or reordered. Text +matching degrades gracefully: a near miss still scores, and the score is what +decides whether firing is safe or the operator has to look. +""" + +import difflib +import re +import tomllib +import unicodedata +from dataclasses import dataclass, field +from pathlib import Path + +from fastform.parse import BLOCKER_TYPES, CHOICE_TYPES, FormSpec, Question + +DEFAULT_BANK_PATH = Path(__file__).resolve().parent.parent / "config" / "answers.toml" + +#: Below this, a fuzzy hit is treated as no match at all. +FUZZY_FLOOR = 0.6 + +_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE) + + +@dataclass +class FilledForm: + pairs: list[tuple[str, str]] = field(default_factory=list) + gaps: list[Question] = field(default_factory=list) + blockers: list[Question] = field(default_factory=list) + confidence: float = 1.0 + + +def load_bank(path: Path | None = None) -> list[dict]: + path = path or DEFAULT_BANK_PATH + with path.open("rb") as fh: + return tomllib.load(fh).get("answer", []) + + +def normalize(text: str) -> str: + text = unicodedata.normalize("NFKC", text).casefold() + text = _PUNCT_RE.sub(" ", text) + return " ".join(text.split()) + + +def score_question(question: Question, entry: dict) -> float: + """How well does this bank entry match this question? 0.0 means no match.""" + pattern = entry.get("regex") + if pattern and re.search(pattern, question.text, re.IGNORECASE): + return 1.0 + + target = normalize(question.text) + target_tokens = set(target.split()) + best = 0.0 + + for phrase in entry.get("match", []): + candidate = normalize(phrase) + if not candidate: + continue + if candidate == target: + best = max(best, 0.95) + continue + if set(candidate.split()) <= target_tokens: + best = max(best, 0.8) + continue + ratio = difflib.SequenceMatcher(None, candidate, target).ratio() + if ratio >= FUZZY_FLOOR: + best = max(best, ratio) + + return best + + +def _resolve_choice(value: str, options: list[str]) -> str | None: + """A choice answer must be one of the offered options, or it will be rejected.""" + if not options: + return value + normalized = {normalize(o): o for o in options} + return normalized.get(normalize(value)) + + +def fill(spec: FormSpec, bank: list[dict]) -> FilledForm: + filled = FilledForm() + scores: list[float] = [] + + for question in spec.questions: + if question.type in BLOCKER_TYPES: + filled.blockers.append(question) + continue + + best_score, best_value = 0.0, None + for entry in bank: + score = score_question(question, entry) + if score > best_score: + best_score, best_value = score, entry["value"] + + if best_value is not None and question.type in CHOICE_TYPES: + best_value = _resolve_choice(best_value, question.options) + if best_value is None: + best_score = 0.0 + + if best_value is not None and best_score > 0.0: + filled.pairs.append((question.entry_id, best_value)) + if question.required: + scores.append(best_score) + elif question.required: + filled.gaps.append(question) + scores.append(0.0) + + filled.confidence = min(scores) if scores else 1.0 + return filled diff --git a/fastform/parse.py b/fastform/parse.py new file mode 100644 index 0000000..e013cc4 --- /dev/null +++ b/fastform/parse.py @@ -0,0 +1,177 @@ +"""Turn a Google Form page into a FormSpec. + +Everything comes out of the FB_PUBLIC_LOAD_DATA_ blob embedded in the page. +A closed form does not include that blob, which is why NotParseable exists and +why the caller has to treat it as a distinct state rather than an error. +""" + +import html as html_mod +import json +import re +from dataclasses import dataclass, field + +TYPE_SHORT_ANSWER = 0 +TYPE_PARAGRAPH = 1 +TYPE_MULTIPLE_CHOICE = 2 +TYPE_DROPDOWN = 3 +TYPE_CHECKBOXES = 4 +TYPE_LINEAR_SCALE = 5 +TYPE_GRID = 7 +TYPE_SECTION = 8 +TYPE_DATE = 9 +TYPE_TIME = 10 +TYPE_FILE_UPLOAD = 13 + +#: Types the direct-POST path cannot satisfy. Hitting one means browser handoff. +BLOCKER_TYPES = frozenset({TYPE_FILE_UPLOAD, TYPE_GRID}) + +#: Types whose value must match one of the offered options. +CHOICE_TYPES = frozenset({TYPE_MULTIPLE_CHOICE, TYPE_DROPDOWN, TYPE_CHECKBOXES}) + +_LOAD_DATA_RE = re.compile( + rb"FB_PUBLIC_LOAD_DATA_\s*=\s*(\[.*?\]);\s*", re.S +) +_FORM_ID_RE = re.compile(r"/forms/d/e/([^/]+)/") +_HIDDEN_RE = re.compile( + rb']+type="hidden"[^>]+name="([^"]+)"[^>]+value="([^"]*)"' +) +#: Present only when the viewer is signed out of a sign-in-required form. +_SIGN_IN_RE = re.compile(rb'data-sign-in-to-continue="true"') +_VIEWER_EMAIL_RE = re.compile(rb'data-user-email-address="([^"]*)"') + + +class NotParseable(Exception): + """The page carries no form structure — closed, deleted, or not a form.""" + + +@dataclass +class Question: + entry_id: str + text: str + type: int + required: bool + options: list[str] = field(default_factory=list) + + +@dataclass +class FormSpec: + form_id: str + action_path: str + page_count: int + questions: list[Question] + #: Hidden inputs from the page, already HTML-unescaped. Carries `token` and + #: `tag`, both of which the POST is rejected without, and both of which + #: change on every page load. + hidden: dict[str, str] = field(default_factory=dict) + + @property + def page_history(self) -> str: + """The pageHistory value that satisfies every section in one POST.""" + return ",".join(str(i) for i in range(self.page_count)) + + @property + def blockers(self) -> list[Question]: + return [q for q in self.questions if q.type in BLOCKER_TYPES] + + +def form_id_from_url(url: str) -> str: + match = _FORM_ID_RE.search(url) + if not match: + raise ValueError(f"not a published Google Form URL: {url}") + return match.group(1) + + +def extract_load_data(html: bytes) -> list: + match = _LOAD_DATA_RE.search(html) + if not match: + raise NotParseable("FB_PUBLIC_LOAD_DATA_ not found") + return json.loads(match.group(1)) + + +def requires_sign_in(html: bytes) -> bool: + """True when this page is a sign-in-required form viewed while signed out. + + Google Forms does not redirect in this case — it serves HTTP 200 with the + whole question structure present and the inputs disabled. Parsing therefore + succeeds on a signed-out page, so this check is the only thing standing + between a dead cookie and a wasted POST. + """ + return bool(_SIGN_IN_RE.search(html)) + + +def signed_in_email(html: bytes) -> str | None: + """The viewing account's address, or None when signed out. + + This is an HTML attribute Google populates from the session, so it is a far + stronger signal than pattern-matching the body for anything email-shaped. + """ + match = _VIEWER_EMAIL_RE.search(html) + if not match: + return None + value = match.group(1).decode("utf-8", "replace") + return value or None + + +def hidden_inputs(html: bytes) -> dict[str, str]: + """Every hidden form input, HTML-unescaped. + + Unescaping is not optional: `partialResponse` holds JSON whose quotes arrive + as `"`, and re-encoding the escaped form gets the POST rejected with a + 400 even when every field name is correct. + """ + return { + name.decode(): html_mod.unescape(value.decode()) + for name, value in _HIDDEN_RE.findall(html) + } + + +def _items(load_data: list) -> list: + try: + return load_data[1][1] or [] + except (IndexError, TypeError) as exc: + raise NotParseable(f"unexpected load data shape: {exc}") from exc + + +def _options(raw_options) -> list[str]: + out = [] + for opt in raw_options or []: + if opt and opt[0]: + out.append(str(opt[0])) + return out + + +def parse(html: bytes, url: str) -> FormSpec: + load_data = extract_load_data(html) + form_id = form_id_from_url(url) + + questions: list[Question] = [] + page_count = 1 + + for item in _items(load_data): + try: + item_type = item[3] + if item_type == TYPE_SECTION: + page_count += 1 + continue + + title = item[1] or "" + for entry in item[4] or []: + questions.append( + Question( + entry_id=str(entry[0]), + text=title, + type=item_type, + required=bool(entry[2]), + options=_options(entry[1]), + ) + ) + except (IndexError, TypeError, KeyError) as exc: + raise NotParseable(f"unexpected item shape: {exc}") from exc + + return FormSpec( + form_id=form_id, + action_path=f"/forms/d/e/{form_id}/formResponse", + page_count=page_count, + questions=questions, + hidden=hidden_inputs(html), + ) diff --git a/fastform/transport.py b/fastform/transport.py new file mode 100644 index 0000000..fd61978 --- /dev/null +++ b/fastform/transport.py @@ -0,0 +1,88 @@ +"""Hold a connection open so sending costs a socket write, not a handshake. + +A cold DNS + TCP + TLS setup to docs.google.com on residential internet is +150-350ms and highly variable — bigger and jitterier than anything else in the +path. Connecting early and heartbeating turns that into zero. +""" + +import http.client +import time +from dataclasses import dataclass + +from fastform.compile import RequestBlob +from fastform.cookies import USER_AGENT + + +@dataclass +class Response: + status: int + location: str | None + body: bytes + elapsed_ms: float + + +class WarmConnection: + """One pre-established HTTP connection, reused across requests.""" + + def __init__( + self, + host: str, + port: int | None = None, + timeout: float = 15.0, + use_tls: bool = True, + ): + self.host = host + self.port = port + self.timeout = timeout + self.use_tls = use_tls + self._conn: http.client.HTTPConnection | None = None + + def connect(self) -> None: + if self._conn is not None: + self._conn.close() + cls = ( + http.client.HTTPSConnection if self.use_tls else http.client.HTTPConnection + ) + self._conn = cls(self.host, port=self.port, timeout=self.timeout) + self._conn.connect() + + def close(self) -> None: + if self._conn is not None: + self._conn.close() + self._conn = None + + def _request(self, method, path, body, headers) -> Response: + if self._conn is None: + raise RuntimeError("connect() must be called before sending") + started = time.perf_counter_ns() + self._conn.request(method, path, body=body, headers=headers) + resp = self._conn.getresponse() + payload = resp.read() # must drain fully or the connection cannot be reused + elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000 + return Response( + status=resp.status, + location=resp.getheader("Location"), + body=payload, + elapsed_ms=elapsed_ms, + ) + + def get(self, path: str, cookie: str) -> Response: + return self._request( + "GET", path, None, {"Cookie": cookie, "User-Agent": USER_AGENT} + ) + + def send(self, blob: RequestBlob) -> Response: + return self._request("POST", blob.path, blob.body, blob.headers) + + def heartbeat(self, path: str, cookie: str) -> bool: + """Keep the socket and any NAT mapping alive. Reconnects on failure.""" + try: + self.get(path, cookie) + return True + except (OSError, http.client.HTTPException): + self.close() + try: + self.connect() + return True + except OSError: + return False diff --git a/fixtures/README.md b/fixtures/README.md new file mode 100644 index 0000000..d7c3490 --- /dev/null +++ b/fixtures/README.md @@ -0,0 +1,36 @@ +# Fixtures + +Captured artifacts used as test data. Nothing here is generated by code. + +- `f1-viewform.html` — F1 form page, signed in. Input for parse tests. +- `f1-manual-submit.curl.txt` — DevTools "copy as cURL" of a real manual + submission to F1. Ground truth for the Task 8 request diff. Untracked: + contains cookies. + +Response fixtures (`resp-*.bin`) are captured in Task 9. + +## Response fixtures + +Captured from real submissions to F1. These are ground truth for classify.py. + +- `resp-success.bin` — HTTP 200 confirmation page. Contains `FB_PUBLIC_LOAD_DATA_` + (so that marker proves nothing), no ``, class `vHW8K` wrapping + "Your response has been recorded." +- `resp-rejected.bin` — HTTP 400, form re-rendered: ``, + `role="listitem"` x3, `name="token"` all present. + +Not captured, because the states do not exist: + +- **already-responded** — "Limit to 1 response" does not block a direct POST. + With it enabled and prior responses from the same account, the GET returns an + ordinary form and the POST records a new row. +- **closed** — a closed form answers 302 to `/closedform` before any body worth + keeping. Detection is the redirect target, not a page marker. + +## Synthetic fixtures (f3, f4) + +`f3-viewform.html` and `f4-viewform.html` are SYNTHETIC, hand-built from the +FB_PUBLIC_LOAD_DATA_ structure verified against the real F1 page. They exist so +parse.py's multi-section and file-upload tests can run without creating real +F3/F4 Google Forms (human browser work). Building the real forms remains a +deferred fidelity check; the parser cannot tell the difference. diff --git a/fixtures/f1-viewform.html b/fixtures/f1-viewform.html new file mode 100644 index 0000000..89732e6 --- /dev/null +++ b/fixtures/f1-viewform.html @@ -0,0 +1 @@ +FastForm F1
FastForm F1
student@example.com Switch account
Email *
What is your full name? *
What is your email address?
 *
What is your phone number?
 *
Submit
Clear form
Never submit passwords through Google Forms.
This form was created outside of your domain. - Contact form owner - Terms of Service - Privacy Policy

Does this form look suspicious? Report

Google Forms
\ No newline at end of file diff --git a/fixtures/f3-viewform.html b/fixtures/f3-viewform.html new file mode 100644 index 0000000..de27e2d --- /dev/null +++ b/fixtures/f3-viewform.html @@ -0,0 +1 @@ +Synthetic \ No newline at end of file diff --git a/fixtures/f4-viewform.html b/fixtures/f4-viewform.html new file mode 100644 index 0000000..39e8b98 --- /dev/null +++ b/fixtures/f4-viewform.html @@ -0,0 +1 @@ +Synthetic \ No newline at end of file diff --git a/fixtures/resp-closed.bin b/fixtures/resp-closed.bin new file mode 100644 index 0000000..24549ef --- /dev/null +++ b/fixtures/resp-closed.bin @@ -0,0 +1 @@ +FastForm F1
FastForm F1
The form FastForm F1 is no longer accepting responses.
Try contacting the owner of the form if you think this is a mistake.
This form was created outside of your domain. - Contact form owner - Terms of Service - Privacy Policy

Does this form look suspicious? Report

Google Forms
\ No newline at end of file diff --git a/fixtures/resp-rejected.bin b/fixtures/resp-rejected.bin new file mode 100644 index 0000000..4f4afa7 --- /dev/null +++ b/fixtures/resp-rejected.bin @@ -0,0 +1 @@ +FastForm F1
FastForm F1
student@example.com Switch account
Email *
What is your full name? *
What is your email address?
 *
What is your phone number?
 *
Submit
Clear form
Never submit passwords through Google Forms.
This form was created outside of your domain. - Contact form owner - Terms of Service - Privacy Policy

Does this form look suspicious? Report

Google Forms
\ No newline at end of file diff --git a/fixtures/resp-success.bin b/fixtures/resp-success.bin new file mode 100644 index 0000000..a92f95b --- /dev/null +++ b/fixtures/resp-success.bin @@ -0,0 +1 @@ +FastForm F1
FastForm F1
Your response has been recorded.
Submit another response
This form was created outside of your domain. - Contact form owner - Terms of Service - Privacy Policy

Does this form look suspicious? Report

Google Forms
\ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_classify.py b/tests/test_classify.py new file mode 100644 index 0000000..06c3156 --- /dev/null +++ b/tests/test_classify.py @@ -0,0 +1,77 @@ +import unittest +from pathlib import Path + +from fastform.classify import Outcome, classify + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" + + +class TestClassifyRealFixtures(unittest.TestCase): + """Ground truth: bodies captured from real submissions to F1. + + resp-success.bin is the case an earlier classifier got wrong (reported + REJECTED on a recorded submission), so it is the most important row here. + """ + + def test_success_is_recorded(self): + body = (FIXTURES / "resp-success.bin").read_bytes() + self.assertEqual(classify(200, None, body), Outcome.RECORDED) + + def test_rejected_is_rejected(self): + body = (FIXTURES / "resp-rejected.bin").read_bytes() + self.assertEqual(classify(400, None, body), Outcome.REJECTED) + + def test_closed_landing_page_is_closed(self): + body = (FIXTURES / "resp-closed.bin").read_bytes() + self.assertEqual(classify(200, None, body), Outcome.CLOSED) + + +class TestClassifyRedirects(unittest.TestCase): + def test_302_to_closedform_is_closed(self): + self.assertEqual( + classify(302, "https://docs.google.com/forms/u/0/d/e/X/closedform", b""), + Outcome.CLOSED, + ) + + def test_302_to_accounts_is_auth_fail(self): + self.assertEqual( + classify(302, "https://accounts.google.com/ServiceLogin", b""), + Outcome.AUTH_FAIL, + ) + + def test_other_redirect_is_unknown(self): + self.assertEqual( + classify(302, "https://docs.google.com/forms/d/e/X/viewform", b""), + Outcome.UNKNOWN, + ) + + +class TestClassifyStructural(unittest.TestCase): + def test_signed_out_200_is_auth_fail_not_rejected(self): + # A signed-out sign-in-required form is served as 200 with the whole + # form. Without the sign-in check this reaches the form-element branch + # and reports REJECTED, sending the operator after a phantom payload bug. + body = ( + b'
' + b"" + b'
' + ) + self.assertEqual(classify(200, None, body), Outcome.AUTH_FAIL) + + def test_form_element_on_200_is_rejected(self): + body = b'' + self.assertEqual(classify(200, None, body), Outcome.REJECTED) + + def test_terminal_page_without_markers_is_unknown(self): + # No form, but no recorded marker either: never guess RECORDED. + self.assertEqual(classify(200, None, b"something else"), Outcome.UNKNOWN) + + def test_server_error_is_unknown(self): + self.assertEqual(classify(500, None, b"oops"), Outcome.UNKNOWN) + + def test_empty_body_is_unknown(self): + self.assertEqual(classify(200, None, b""), Outcome.UNKNOWN) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_compile.py b/tests/test_compile.py new file mode 100644 index 0000000..8e1495d --- /dev/null +++ b/tests/test_compile.py @@ -0,0 +1,112 @@ +import unittest +import urllib.parse + +from fastform import compile as compile_mod +from fastform.match import FilledForm +from fastform.parse import FormSpec + + +def spec(pages=1, hidden=None): + return FormSpec( + form_id="ABC", action_path="/forms/d/e/ABC/formResponse", + page_count=pages, questions=[], + hidden=hidden if hidden is not None else {"token": "TOK", "tag": "TAG"}, + ) + + +class TestNewFbzx(unittest.TestCase): + def test_is_a_signed_integer_string(self): + value = compile_mod.new_fbzx() + int(value) + + def test_differs_between_calls(self): + self.assertNotEqual(compile_mod.new_fbzx(), compile_mod.new_fbzx()) + + +class TestBuild(unittest.TestCase): + def _fields(self, blob): + return dict(urllib.parse.parse_qsl(blob.body)) + + def test_entry_pairs_are_prefixed(self): + filled = FilledForm(pairs=[("123", "Alice")]) + blob = compile_mod.build(spec(), filled, "SID=x", fbzx="-1") + self.assertEqual(self._fields(blob)["entry.123"], "Alice") + + def test_includes_fvv_and_timestamp(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + fields = self._fields(blob) + self.assertEqual(fields["fvv"], "1") + self.assertEqual(fields["submissionTimestamp"], "-1") + + def test_page_history_covers_every_section(self): + blob = compile_mod.build(spec(pages=3), FilledForm(), "SID=x", fbzx="-1") + self.assertEqual(self._fields(blob)["pageHistory"], "0,1,2") + + def test_cookie_and_content_type_headers(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=abc", fbzx="-1") + self.assertEqual(blob.headers["Cookie"], "SID=abc") + self.assertEqual( + blob.headers["Content-Type"], "application/x-www-form-urlencoded" + ) + + def test_referer_points_at_the_viewform(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + self.assertEqual( + blob.headers["Referer"], + "https://docs.google.com/forms/d/e/ABC/viewform", + ) + + def test_unicode_values_survive_encoding(self): + filled = FilledForm(pairs=[("1", "กรุงเทพ")]) + blob = compile_mod.build(spec(), filled, "SID=x", fbzx="-1") + self.assertEqual(self._fields(blob)["entry.1"], "กรุงเทพ") + + def test_generates_fbzx_when_not_supplied(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x") + int(self._fields(blob)["fbzx"]) + + def test_content_length_matches_encoded_body(self): + filled = FilledForm(pairs=[("1", "กรุงเทพ")]) + blob = compile_mod.build(spec(), filled, "SID=x", fbzx="-1") + self.assertEqual( + blob.headers["Content-Length"], + str(len(blob.body.encode("utf-8"))), + ) + + def test_carries_token_and_tag_from_the_page(self): + # Measured: dropping either gets the POST rejected with 400. + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + fields = self._fields(blob) + self.assertEqual(fields["token"], "TOK") + self.assertEqual(fields["tag"], "TAG") + + def test_missing_token_raises_rather_than_posting_garbage(self): + with self.assertRaises(compile_mod.MissingToken): + compile_mod.build( + spec(hidden={"tag": "TAG"}), FilledForm(), "SID=x", fbzx="-1" + ) + + def test_missing_tag_raises(self): + with self.assertRaises(compile_mod.MissingToken): + compile_mod.build( + spec(hidden={"token": "TOK"}), FilledForm(), "SID=x", fbzx="-1" + ) + + def test_email_included_when_supplied(self): + blob = compile_mod.build( + spec(), FilledForm(), "SID=x", email="a@b.test", fbzx="-1" + ) + self.assertEqual(self._fields(blob)["emailAddress"], "a@b.test") + + def test_email_omitted_when_not_supplied(self): + blob = compile_mod.build(spec(), FilledForm(), "SID=x", fbzx="-1") + self.assertNotIn("emailAddress", self._fields(blob)) + + def test_page_fbzx_preferred_over_generated(self): + s = spec(hidden={"token": "TOK", "tag": "TAG", "fbzx": "42"}) + blob = compile_mod.build(s, FilledForm(), "SID=x") + self.assertEqual(self._fields(blob)["fbzx"], "42") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cookies.py b/tests/test_cookies.py new file mode 100644 index 0000000..3716a24 --- /dev/null +++ b/tests/test_cookies.py @@ -0,0 +1,247 @@ +import sqlite3 +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from fastform import cookies + + +class TestLoad(unittest.TestCase): + def _write(self, text): + tmp = Path(tempfile.mkdtemp()) / "cookies.txt" + tmp.write_text(text, encoding="utf-8") + return tmp + + def test_returns_bare_value(self): + path = self._write("SID=abc; HSID=def\n") + self.assertEqual(cookies.load(path), "SID=abc; HSID=def") + + def test_strips_cookie_prefix(self): + path = self._write("Cookie: SID=abc; HSID=def") + self.assertEqual(cookies.load(path), "SID=abc; HSID=def") + + def test_strips_lowercase_prefix(self): + path = self._write("cookie: SID=abc") + self.assertEqual(cookies.load(path), "SID=abc") + + def test_empty_file_raises(self): + path = self._write(" \n") + with self.assertRaises(ValueError): + cookies.load(path) + + def test_missing_file_raises(self): + path = Path(tempfile.mkdtemp()) / "nope.txt" + with self.assertRaises(FileNotFoundError): + cookies.load(path) + + +def _fake_connection(status, location=None, body=b""): + """Build a stand-in for http.client.HTTPSConnection's instance. + + Mimics the subset of the interface verify() touches: request(), + getresponse() (returning an object with .status, .getheader(), and + .read()), and close(). + """ + response = MagicMock() + response.status = status + response.getheader.side_effect = ( + lambda name, default=None: location if name == "Location" else default + ) + response.read.return_value = body + + conn = MagicMock() + conn.getresponse.return_value = response + return conn + + +class TestVerify(unittest.TestCase): + def _patch_connection(self, status, location=None, body=b""): + """Patch http.client.HTTPSConnection as seen from fastform.cookies + so verify() can never open a real socket, and return the fake + connection instance it will receive. + """ + conn = _fake_connection(status, location, body) + patcher = patch( + "fastform.cookies.http.client.HTTPSConnection", return_value=conn + ) + self.addCleanup(patcher.stop) + patcher.start() + return conn + + def test_200_with_email_is_signed_in(self): + self._patch_connection(200, body=b"Signed in as jane.doe@example.com today") + signed_in, email = cookies.verify("SID=abc") + self.assertTrue(signed_in) + self.assertEqual(email, "jane.doe@example.com") + + def test_200_without_email_is_signed_in_with_none(self): + self._patch_connection(200, body=b"no email on this page") + signed_in, email = cookies.verify("SID=abc") + self.assertTrue(signed_in) + self.assertIsNone(email) + + def test_302_to_accounts_google_is_signed_out(self): + self._patch_connection( + 302, location="https://accounts.google.com/ServiceLogin", body=b"" + ) + signed_in, email = cookies.verify("SID=abc") + self.assertFalse(signed_in) + self.assertIsNone(email) + + def test_302_to_other_location_is_also_signed_out(self): + # verify() only special-cases redirects whose Location contains + # accounts.google.com. A redirect anywhere else does not match that + # branch, but it still isn't status 200, so it falls through to the + # generic "status != 200" check and is reported as signed out -- + # not because the redirect target was actually inspected. + self._patch_connection( + 302, + location="https://docs.google.com/forms/u/0/somewhere-else", + body=b"", + ) + signed_in, email = cookies.verify("SID=abc") + self.assertFalse(signed_in) + self.assertIsNone(email) + + def test_non_200_non_redirect_status_is_signed_out(self): + self._patch_connection(500, body=b"Internal Server Error") + signed_in, email = cookies.verify("SID=abc") + self.assertFalse(signed_in) + self.assertIsNone(email) + + def test_connection_is_closed(self): + conn = self._patch_connection(200, body=b"no email here") + cookies.verify("SID=abc") + conn.close.assert_called_once() + + +class TestDeriveKey(unittest.TestCase): + def test_key_is_16_bytes(self): + self.assertEqual(len(cookies.derive_key("abc")), 16) + + def test_deterministic(self): + self.assertEqual(cookies.derive_key("abc"), cookies.derive_key("abc")) + + def test_differs_per_secret(self): + self.assertNotEqual(cookies.derive_key("abc"), cookies.derive_key("abd")) + + +class TestDecryptValue(unittest.TestCase): + """Round-trip against openssl so the test proves the real cipher path.""" + + KEY = cookies.derive_key("test-secret") + + def _encrypt(self, plaintext: bytes) -> bytes: + proc = subprocess.run( + ["openssl", "enc", "-aes-128-cbc", "-K", self.KEY.hex(), + "-iv", (b" " * 16).hex()], + input=plaintext, capture_output=True, check=True, + ) + return b"v10" + proc.stdout + + def test_round_trip(self): + blob = self._encrypt(b"SID=hello-world") + self.assertEqual(cookies.decrypt_value(blob, self.KEY), "SID=hello-world") + + def test_unencrypted_blob_returns_none(self): + self.assertIsNone(cookies.decrypt_value(b"plaintext", self.KEY)) + + def test_empty_blob_returns_none(self): + self.assertIsNone(cookies.decrypt_value(b"", self.KEY)) + + def test_wrong_key_does_not_raise(self): + blob = self._encrypt(b"SID=hello-world") + other = cookies.derive_key("different") + result = cookies.decrypt_value(blob, other) + self.assertNotEqual(result, "SID=hello-world") + + +class TestListProfiles(unittest.TestCase): + def _profile_dir(self, name, rows): + root = Path(tempfile.mkdtemp()) + prof = root / name + prof.mkdir() + con = sqlite3.connect(prof / "Cookies") + con.execute( + "CREATE TABLE cookies (host_key TEXT, name TEXT, encrypted_value BLOB)" + ) + con.executemany("INSERT INTO cookies VALUES (?,?,?)", rows) + con.commit() + con.close() + return root + + def test_counts_google_cookies(self): + root = self._profile_dir( + "Profile 1", + [(".google.com", "SID", b"v10x"), ("example.com", "other", b"v10y")], + ) + profiles = cookies.list_profiles(root) + self.assertEqual(len(profiles), 1) + self.assertEqual(profiles[0].name, "Profile 1") + self.assertEqual(profiles[0].google_cookies, 1) + + def test_skips_profiles_without_a_cookie_db(self): + root = Path(tempfile.mkdtemp()) + (root / "Profile 9").mkdir() + self.assertEqual(cookies.list_profiles(root), []) + + def test_missing_root_returns_empty(self): + self.assertEqual( + cookies.list_profiles(Path(tempfile.mkdtemp()) / "nope"), [] + ) + + +class TestDecryptDomainHashPrefix(unittest.TestCase): + """Newer Chrome prefixes the plaintext with 32 bytes of domain hash. + + The reviewer flagged this branch as untested. A wrong strip corrupts every + cookie value while still looking plausible, so it is worth a real round-trip. + """ + + KEY = cookies.derive_key("hash-prefix-secret") + + def _encrypt(self, plaintext: bytes) -> bytes: + proc = subprocess.run( + ["openssl", "enc", "-aes-128-cbc", "-K", self.KEY.hex(), + "-iv", (b" " * 16).hex()], + input=plaintext, capture_output=True, check=True, + ) + return b"v10" + proc.stdout + + def test_32_byte_binary_prefix_is_stripped(self): + # A real domain hash is 32 non-printable bytes ahead of the value. + prefix = bytes(range(32)) + blob = self._encrypt(prefix + b"SID=value-after-hash") + self.assertEqual( + cookies.decrypt_value(blob, self.KEY), "SID=value-after-hash" + ) + + def test_short_printable_value_is_not_mistaken_for_a_prefix(self): + # A value under 32 bytes must survive intact — no prefix to strip. + blob = self._encrypt(b"SID=short") + self.assertEqual(cookies.decrypt_value(blob, self.KEY), "SID=short") + + +class TestCookieHeaderRequiresProfile(unittest.TestCase): + def test_profile_is_a_required_positional_argument(self): + with self.assertRaises(TypeError): + cookies.cookie_header_from_chrome() # noqa - intentionally missing arg + + def test_unknown_profile_raises_chrome_cookie_error(self): + real = cookies.list_profiles + + def fake(root=None): + return [cookies.ChromeProfile("Profile 1", Path("/x"), 5)] + + cookies.list_profiles = fake + try: + with self.assertRaises(cookies.ChromeCookieError): + cookies.cookie_header_from_chrome("Profile 99") + finally: + cookies.list_profiles = real + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_match.py b/tests/test_match.py new file mode 100644 index 0000000..9c4be0e --- /dev/null +++ b/tests/test_match.py @@ -0,0 +1,105 @@ +import unittest + +from fastform import match +from fastform.parse import ( + TYPE_FILE_UPLOAD, + TYPE_MULTIPLE_CHOICE, + TYPE_SHORT_ANSWER, + FormSpec, + Question, +) + + +def spec(*questions): + return FormSpec( + form_id="X", action_path="/p", page_count=1, questions=list(questions) + ) + + +BANK = [ + {"match": ["full name", "your name"], "value": "Test Person"}, + {"match": ["email address"], "regex": "e-?mail", "value": "test@example.com"}, + {"match": ["which city"], "value": "Bangkok"}, +] + + +class TestNormalize(unittest.TestCase): + def test_casefolds_and_strips_punctuation(self): + self.assertEqual(match.normalize("What is your Name?"), "what is your name") + + def test_collapses_whitespace(self): + self.assertEqual(match.normalize("a b\n c"), "a b c") + + +class TestFill(unittest.TestCase): + def test_exact_text_match_fills(self): + s = spec(Question("111", "full name", TYPE_SHORT_ANSWER, True)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, [("111", "Test Person")]) + self.assertEqual(filled.gaps, []) + + def test_regex_wins_and_scores_one(self): + s = spec(Question("222", "Your E-Mail", TYPE_SHORT_ANSWER, True)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, [("222", "test@example.com")]) + self.assertEqual(filled.confidence, 1.0) + + def test_unmatched_required_becomes_a_gap(self): + s = spec(Question("333", "Favourite dinosaur", TYPE_SHORT_ANSWER, True)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, []) + self.assertEqual([q.entry_id for q in filled.gaps], ["333"]) + + def test_unmatched_optional_is_omitted_not_a_gap(self): + s = spec(Question("444", "Favourite dinosaur", TYPE_SHORT_ANSWER, False)) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, []) + self.assertEqual(filled.gaps, []) + + def test_choice_value_must_exist_in_options(self): + s = spec( + Question( + "555", + "Which city?", + TYPE_MULTIPLE_CHOICE, + True, + options=["Bangkok Metropolitan Region", "Phuket"], + ) + ) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, []) + self.assertEqual([q.entry_id for q in filled.gaps], ["555"]) + + def test_choice_value_present_in_options_fills(self): + s = spec( + Question( + "666", + "Which city?", + TYPE_MULTIPLE_CHOICE, + True, + options=["Bangkok", "Phuket"], + ) + ) + filled = match.fill(s, BANK) + self.assertEqual(filled.pairs, [("666", "Bangkok")]) + + def test_blockers_are_reported_separately(self): + s = spec(Question("777", "Upload", TYPE_FILE_UPLOAD, False)) + filled = match.fill(s, BANK) + self.assertEqual([q.entry_id for q in filled.blockers], ["777"]) + + def test_confidence_is_min_over_required_questions(self): + s = spec( + Question("111", "full name", TYPE_SHORT_ANSWER, True), + Question("333", "Favourite dinosaur", TYPE_SHORT_ANSWER, True), + ) + filled = match.fill(s, BANK) + self.assertEqual(filled.confidence, 0.0) + + def test_confidence_is_one_when_no_required_questions(self): + s = spec(Question("444", "Anything", TYPE_SHORT_ANSWER, False)) + self.assertEqual(match.fill(s, BANK).confidence, 1.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parse.py b/tests/test_parse.py new file mode 100644 index 0000000..c1398a4 --- /dev/null +++ b/tests/test_parse.py @@ -0,0 +1,151 @@ +import unittest +from pathlib import Path + +from fastform import parse + +FIXTURES = Path(__file__).resolve().parent.parent / "fixtures" + +F1_URL = "https://docs.google.com/forms/d/e/FAKEID123/viewform" + + +class TestFormIdFromUrl(unittest.TestCase): + def test_extracts_published_id(self): + self.assertEqual( + parse.form_id_from_url( + "https://docs.google.com/forms/d/e/1FAIpQLSabc/viewform?usp=sf_link" + ), + "1FAIpQLSabc", + ) + + def test_rejects_non_form_url(self): + with self.assertRaises(ValueError): + parse.form_id_from_url("https://example.com/hello") + + +class TestParseF1(unittest.TestCase): + @classmethod + def setUpClass(cls): + html = (FIXTURES / "f1-viewform.html").read_bytes() + cls.spec = parse.parse(html, F1_URL) + + def test_finds_three_questions(self): + self.assertEqual(len(self.spec.questions), 3) + + def test_question_text_is_readable(self): + texts = [q.text for q in self.spec.questions] + self.assertIn("What is your full name?", texts) + + def test_entry_ids_are_numeric_strings(self): + for q in self.spec.questions: + self.assertTrue(q.entry_id.isdigit(), q.entry_id) + + def test_all_required(self): + self.assertTrue(all(q.required for q in self.spec.questions)) + + def test_single_page(self): + self.assertEqual(self.spec.page_count, 1) + self.assertEqual(self.spec.page_history, "0") + + def test_action_path(self): + self.assertEqual( + self.spec.action_path, "/forms/d/e/FAKEID123/formResponse" + ) + + def test_no_blockers(self): + self.assertEqual(self.spec.blockers, []) + + def test_hidden_inputs_carry_token_and_tag(self): + self.assertIn("token", self.spec.hidden) + self.assertIn("tag", self.spec.hidden) + self.assertTrue(self.spec.hidden["token"]) + + def test_hidden_values_are_html_unescaped(self): + # partialResponse holds JSON, so it contains quotes that the page + # escapes as ". Leaving them escaped 400s the POST. + self.assertNotIn(""", self.spec.hidden.get("partialResponse", "")) + + +class TestSignInDetection(unittest.TestCase): + """Google Forms returns 200 for a signed-out sign-in-required form, with the + question structure fully present. Only an attribute distinguishes the two.""" + + SIGNED_OUT = b'
' + SIGNED_IN = b'
' + + def test_detects_signed_out(self): + self.assertTrue(parse.requires_sign_in(self.SIGNED_OUT)) + + def test_detects_signed_in(self): + self.assertFalse(parse.requires_sign_in(self.SIGNED_IN)) + + def test_extracts_email_when_signed_in(self): + self.assertEqual( + parse.signed_in_email(self.SIGNED_IN), "someone@example.com" + ) + + def test_no_email_when_signed_out(self): + self.assertIsNone(parse.signed_in_email(self.SIGNED_OUT)) + + +class TestHiddenInputs(unittest.TestCase): + def test_unescapes_values(self): + html = b'' + self.assertEqual(parse.hidden_inputs(html), {"x": 'a"b&c'}) + + def test_ignores_non_hidden_inputs(self): + html = b'' + self.assertEqual(parse.hidden_inputs(html), {}) + + +class TestParseF3(unittest.TestCase): + @classmethod + def setUpClass(cls): + html = (FIXTURES / "f3-viewform.html").read_bytes() + cls.spec = parse.parse(html, F1_URL) + + def test_three_pages(self): + self.assertEqual(self.spec.page_count, 3) + self.assertEqual(self.spec.page_history, "0,1,2") + + def test_choice_options_captured(self): + city = next(q for q in self.spec.questions if "city" in q.text.lower()) + self.assertIn("Bangkok", city.options) + self.assertIn("Chiang Mai", city.options) + + def test_optional_question_detected(self): + optional = [q for q in self.spec.questions if not q.required] + self.assertTrue(optional, "expected at least one optional question") + + +class TestParseF4(unittest.TestCase): + @classmethod + def setUpClass(cls): + html = (FIXTURES / "f4-viewform.html").read_bytes() + cls.spec = parse.parse(html, F1_URL) + + def test_file_upload_is_a_blocker(self): + self.assertEqual(len(self.spec.blockers), 1) + self.assertEqual(self.spec.blockers[0].type, parse.TYPE_FILE_UPLOAD) + + +class TestParseFailures(unittest.TestCase): + def test_missing_load_data_raises(self): + with self.assertRaises(parse.NotParseable): + parse.parse(b"form is closed", F1_URL) + + def test_structurally_short_item_raises_not_parseable(self): + # item is [123, "t"] here -- far too short to unpack as + # [id, title, description, type, entries, ...]. The blob itself is + # present and well-formed, so this must not be confused with the + # "no FB_PUBLIC_LOAD_DATA_" case; it's a distinct malformed-item case + # that must still surface as NotParseable, not a bare IndexError. + html = ( + b'' + ) + with self.assertRaises(parse.NotParseable): + parse.parse(html, F1_URL) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_transport.py b/tests/test_transport.py new file mode 100644 index 0000000..11647ea --- /dev/null +++ b/tests/test_transport.py @@ -0,0 +1,117 @@ +import http.server +import threading +import unittest + +from fastform.compile import RequestBlob +from fastform.transport import Response, WarmConnection + + +class _Handler(http.server.BaseHTTPRequestHandler): + # HTTP/1.1 so the server keeps the connection alive between requests. Under + # the default HTTP/1.0 the server closes after every response, http.client + # silently opens a fresh socket, and test_connection_is_reusable would pass + # even if WarmConnection failed to drain and reuse — i.e. it could not fail. + # HTTP/1.1 keep-alive requires a correct Content-Length on every response. + protocol_version = "HTTP/1.1" + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + payload = b"echo:" + body + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def do_GET(self): + payload = b"ok" + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + +class TestWarmConnection(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.server = http.server.HTTPServer(("127.0.0.1", 0), _Handler) + cls.port = cls.server.server_address[1] + cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) + cls.thread.start() + + @classmethod + def tearDownClass(cls): + cls.server.shutdown() + cls.server.server_close() + + def _conn(self, timeout=15.0): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False, timeout=timeout) + conn.connect() + self.addCleanup(conn.close) + return conn + + def test_send_returns_body_and_status(self): + conn = self._conn() + blob = RequestBlob( + host="127.0.0.1", path="/x", body="a=1", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + resp = conn.send(blob) + self.assertIsInstance(resp, Response) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.body, b"echo:a=1") + + def test_records_elapsed_time(self): + conn = self._conn() + blob = RequestBlob(host="127.0.0.1", path="/x", body="a=1", headers={}) + self.assertGreater(conn.send(blob).elapsed_ms, 0.0) + + def test_connection_is_reusable(self): + # A real reuse guard: the SAME underlying socket must serve both + # requests. If WarmConnection stopped draining the body, http.client + # could not reuse the keep-alive socket; the short timeout makes that + # regression fail fast here instead of hanging the suite. + conn = self._conn(timeout=5.0) + blob = RequestBlob(host="127.0.0.1", path="/x", body="a=1", headers={}) + self.assertEqual(conn.send(blob).status, 200) + first_sock = id(conn._conn.sock) + self.assertEqual(conn.send(blob).status, 200) + self.assertEqual(id(conn._conn.sock), first_sock, + "second request opened a new socket — connection not reused") + + def test_heartbeat_keeps_it_alive(self): + conn = self._conn() + self.assertTrue(conn.heartbeat("/ping", "SID=x")) + + def test_send_before_connect_raises(self): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False) + blob = RequestBlob(host="127.0.0.1", path="/x", body="", headers={}) + with self.assertRaises(RuntimeError): + conn.send(blob) + + def test_get_before_connect_raises(self): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False) + with self.assertRaises(RuntimeError): + conn.get("/x", "SID=x") + + def test_connect_twice_closes_the_first_socket(self): + conn = WarmConnection("127.0.0.1", port=self.port, use_tls=False) + conn.connect() + self.addCleanup(conn.close) + conn.send(RequestBlob(host="127.0.0.1", path="/x", body="a=1", headers={})) + first = conn._conn.sock + conn.connect() # must close `first` rather than leak it + self.assertEqual(first.fileno(), -1, "first socket was leaked on reconnect") + # the fresh connection still works + self.assertEqual( + conn.send(RequestBlob(host="127.0.0.1", path="/x", body="b=2", headers={})).status, + 200, + ) + + +if __name__ == "__main__": + unittest.main()