From 9b73c18b61ebe66d98679a399dbf461e3561b5a8 Mon Sep 17 00:00:00 2001 From: Maksym Prokopov Date: Fri, 31 Jul 2026 09:17:50 +0200 Subject: [PATCH 1/2] Add a consumption test for the compiled wiki MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repo verifies that md2okf produces a well-formed wiki. Nothing verified the wiki was any use, which is a different claim and the one OKF's own goals make ("inform how consumption agents should read and traverse it"). The test gives an agent a badly written paragraph and nothing but the wiki: a new pi-consume service mounts okf/ read-only and does not mount md/ at all, so the source documents are out of reach. Each reply must end in a json block naming its changes and the page each came from. Grading is three string/filesystem assertions, no model in the loop: the edit landed, the citation resolves, and the cited page really contains the ruling. The third is the point. Without it an agent could produce the right edit from memory, attach a plausible path, and score full marks having never opened the wiki. The cases lean on rulings a model cannot guess — the wiki requires the % sign where most guides spell out "per cent", and forbids sentence-initial "Hopefully" while conceding it is not a grammatical error. Answering from priors gets those wrong, which is what gives the test the ability to fail. Provider and model are passed as flags rather than read from settings.json, so a run states which model it exercised and leaves repo config untouched. --- .gitignore | 4 +- .../2026-07-31-okf-consumption-test-design.md | 138 ++++++++++++++++++ .../agent/skills/apply-house-style/SKILL.md | 57 ++++++++ pi/container/compose.yaml | 48 ++++++ .../agent/skills/apply-house-style/SKILL.md | 57 ++++++++ scripts/test-house-style.sh | 86 +++++++++++ tests/house-style/cases.json | 101 +++++++++++++ tests/house-style/grade.py | 136 +++++++++++++++++ 8 files changed, 626 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/specs/2026-07-31-okf-consumption-test-design.md create mode 100644 pi/container/agent/skills/apply-house-style/SKILL.md create mode 100644 pi/sandbox/files/home/.pi/agent/skills/apply-house-style/SKILL.md create mode 100755 scripts/test-house-style.sh create mode 100644 tests/house-style/cases.json create mode 100755 tests/house-style/grade.py diff --git a/.gitignore b/.gitignore index 2813865..6c07e39 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,6 @@ md/* # okf output folder okf/* !okf/ -!okf/.okflintrc.json \ No newline at end of file +!okf/.okflintrc.json +# consumption-test transcripts (generated) +tests/house-style/out/ diff --git a/docs/superpowers/specs/2026-07-31-okf-consumption-test-design.md b/docs/superpowers/specs/2026-07-31-okf-consumption-test-design.md new file mode 100644 index 0000000..e1db1d8 --- /dev/null +++ b/docs/superpowers/specs/2026-07-31-okf-consumption-test-design.md @@ -0,0 +1,138 @@ +# OKF consumption test — design + +**Date:** 2026-07-31 +**Status:** approved, for implementation + +## Problem + +`md2okf` is verified end to end as a *producer*: a 382 KB book compiles into a +38-page OKF wiki that passes `okf-lint`. Nothing verifies the wiki is any +*use*. Completeness and well-formedness are not usefulness, and OKF's own goals +name consumption explicitly: + +> Inform how **consumption agents** should read and traverse it. +> — `SPEC.md`, Goals §2 + +So: can an agent holding only the compiled wiki apply the knowledge in it, and +can we tell the difference between it having done so and it having guessed? + +## The prior-knowledge problem + +This is the crux. A capable model already knows roughly what Economist house +style is. Hand it a badly written paragraph and a wiki, and a good rewrite +proves nothing — it may never have opened a page. + +Two mitigations, both required: + +1. **Arbitrary rulings.** Cases are built on house rulings a model cannot + derive from general knowledge, several of which invert the common + convention. The strongest is `%`: most style guides prefer spelling out + "per cent", and this one requires the sign. A model working from priors gets + it backwards. +2. **Verified citations.** Every change must cite the page it came from, and + grading checks the cited page *actually says so*. A plausible citation to a + page that does not support the claim fails. + +There is deliberately **no A/B control arm** (running each case again with the +wiki absent). It would be the cleanest evidence of contribution, but it doubles +the runs, and the counterintuitive rulings already give the test the ability to +fail when the wiki is unread. Recorded as a known limitation, not an oversight. + +## Isolation + +A second Compose service, `pi-consume`, reusing the existing image: + +| Mount | Compile run | Consumption run | +| --- | --- | --- | +| `md/` | read-only | **absent** | +| `okf/` | read-write | **read-only** | + +`md/` is unmounted so the agent cannot reach the source book — only the compiled +wiki. `okf/` is read-only because consumption must not mutate the artefact under +test; it also means the run needs no writable mount at all. + +The driver passes `--provider` and `--model` explicitly rather than editing +`settings.json`, so the test states which model it exercised and leaves repo +config untouched. + +## Output contract + +The agent writes nothing. Its final message is the result, captured from +`pi -p` stdout, and must end with a fenced `json` block: + +```json +{ + "rewrite": "the corrected paragraph", + "changes": [ + { + "before": "per cent", + "after": "%", + "ruling": "Use the sign % instead of per cent.", + "citation": "/part-2/7-sweating-the-small-stuff-punctuation-mechanics-and-conventions.md" + } + ] +} +``` + +The grader reads the **last** fenced `json` block, so surrounding prose is +harmless. A missing or unparseable block fails the case explicitly rather than +scoring zero silently. + +## Grading + +Three mechanical assertions per case. No LLM judge — every check is a string or +filesystem operation, so the result is deterministic and reproducible. + +1. **applied** — every `expect_present` string appears in `rewrite`, and every + `expect_absent` string does not. +2. **cited** — at least one `changes[].citation` resolves to a file that exists + in the bundle. +3. **grounded** — that cited file contains the case's `grounding` text. + +A case passes only when all three hold. Assertion 3 is what makes hallucinated +citations fail: getting the edit right by luck and inventing a source still +fails the case. + +## Cases + +All eight are verified present in the compiled wiki. `ch6` is +`/part-2/6-confusables-and-cuttables-individual-rulings.md`. + +| # | Ruling | Page | Discriminates because | +| --- | --- | --- | --- | +| 1 | `%` not "per cent" | ch7 | inverts the usual convention | +| 2 | do not verb "impact" | ch9 | house prohibition, not grammar | +| 3 | "due to" modifies nouns only | ch6 | prefers *because of* / *owing to* | +| 4 | "decimate" = a significant proportion | ch6 | not total destruction | +| 5 | "fewer" for countables | ch6 | common error, explicit ruling | +| 6 | no sentence-initial "Hopefully," | ch6 | explicitly *not* a grammar rule | +| 7 | "alibi" ≠ excuse | ch6 | semantic ruling | +| 8 | "compared with" for evaluation | ch6 | fine distinction against *compared to* | + +## Failure taxonomy + +Infrastructure failure must never read as "the wiki is bad". The driver exits: + +- `0` — all cases passed +- `1` — at least one case failed on its assertions (a real result) +- `2` — infrastructure trouble: missing key, gateway error, no wiki, no cases + +## Components + +| Path | Purpose | +| --- | --- | +| `pi/{container,sandbox}/…/skills/apply-house-style/SKILL.md` | the consumption task, one copy per runtime, aligned by hand | +| `tests/house-style/cases.yaml` | fixtures: input, assertions, expected page, grounding | +| `tests/house-style/grade.py` | the three assertions, table output, exit code | +| `scripts/test-house-style.sh` | driver: one Pi run per case, then grade | +| `pi/container/compose.yaml` | adds the `pi-consume` service | + +## Non-goals + +- **Navigability.** The container ships `ripgrep` and `fd`, so the agent will + likely grep rather than traverse `index.md` files. This test says nothing + about whether the index structure works. +- **Prose quality.** Only the specific rulings are graded, not whether the + rewrite reads well. +- **The sandbox runtime.** Implemented for parity, exercised on the container + runtime only, matching how every other result in this repo was obtained. diff --git a/pi/container/agent/skills/apply-house-style/SKILL.md b/pi/container/agent/skills/apply-house-style/SKILL.md new file mode 100644 index 0000000..1233494 --- /dev/null +++ b/pi/container/agent/skills/apply-house-style/SKILL.md @@ -0,0 +1,57 @@ +--- +name: apply-house-style +description: Rewrite a paragraph to follow the house style recorded in the OKF wiki under okf/, citing the page each change came from. Use when a run asks you to apply house style to a passage. +--- + +# Apply house style from the wiki + +You are given one paragraph. Rewrite it so it follows the house style recorded +in the OKF wiki under `okf/`, and say which page each change came from. + +The wiki is **read-only** in this run and there is no source document to fall +back on — `okf/` is all you have, and it is the point. Answer from what the +pages actually say, not from what you already believe about house style. The +two often disagree, and here the wiki wins. + +## Procedure + +1. **Find the rulings.** Search `okf/` for the words and constructions used in + the paragraph. A ruling is usually a short section named after the word it + governs. Read the page before relying on it. +2. **Apply only what you found.** Make a change when a page supports it. Do not + make stylistic improvements the wiki does not ask for — an unsupported change + cannot be cited, and uncited changes are the failure this task looks for. +3. **Leave the rest alone.** Preserve the paragraph's meaning, facts and + figures. You are correcting style, not rewriting the argument. +4. **Report** in the format below. + +## Output format + +End your reply with a fenced `json` block, and put nothing after it: + +````text +```json +{ + "rewrite": "the full corrected paragraph", + "changes": [ + { + "before": "the original wording", + "after": "the corrected wording", + "ruling": "a short quote of what the page says", + "citation": "/part-2/6-confusables-and-cuttables-individual-rulings.md" + } + ] +} +``` +```` + +- `rewrite` — the whole paragraph, corrected. Not a diff, not a fragment. +- `changes` — one entry per change. An empty list is a valid answer if the + wiki genuinely supports no change. +- `ruling` — quote the page, briefly. Do not paraphrase it into something the + page does not say. +- `citation` — a **bundle-absolute** path to a page that exists, rooted at the + wiki root: `/part-2/….md`, not `okf/part-2/….md` and not a relative path. + +Cite the page you actually read. A citation to a page that does not contain the +ruling is worse than making no change at all: it presents a guess as sourced. diff --git a/pi/container/compose.yaml b/pi/container/compose.yaml index d1c64da..7259efb 100644 --- a/pi/container/compose.yaml +++ b/pi/container/compose.yaml @@ -56,5 +56,53 @@ services: - ../../md:/workspace/md:ro - ../../okf:/workspace/okf:rw + # Consumption runtime: reads the compiled wiki, never writes it. Used by + # scripts/test-house-style.sh to check the wiki is usable, not merely + # well-formed. Two differences from `pi` above carry the whole point: + # + # - md/ is NOT mounted, so the agent cannot reach the source documents and + # must answer from okf/ alone. Mount it and the test proves nothing. + # - okf/ is read-only, because the artefact under test must survive the test + # unchanged. Nothing is written, so no writable mount is needed; the + # agent's answer is its stdout. + # + # Neither key is required here — the driver selects a provider with + # --provider/--model, and only that provider's key has to be set. + pi-consume: + build: + context: ../.. + dockerfile: pi/container/Dockerfile + image: pi-container + + environment: + HOME: /home/node + OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-} + LITELLM_API_KEY: ${LITELLM_API_KEY:-} + + network_mode: bridge + cap_drop: + - ALL + security_opt: + - no-new-privileges + read_only: true + user: "1000:1000" + + pids_limit: 512 + mem_limit: 4g + cpus: 2 + + tmpfs: + - /tmp + - /run + - /home/node + + volumes: + - pi-agent-home:/home/node/.pi/agent + - ./agent/settings.json:/home/node/.pi/agent/settings.json:ro + - ./agent/models.json:/home/node/.pi/agent/models.json:ro + - ./agent/AGENTS.md:/home/node/.pi/agent/AGENTS.md:ro + - ./agent/skills:/home/node/.pi/agent/skills:ro + - ../../okf:/workspace/okf:ro + volumes: pi-agent-home: diff --git a/pi/sandbox/files/home/.pi/agent/skills/apply-house-style/SKILL.md b/pi/sandbox/files/home/.pi/agent/skills/apply-house-style/SKILL.md new file mode 100644 index 0000000..1233494 --- /dev/null +++ b/pi/sandbox/files/home/.pi/agent/skills/apply-house-style/SKILL.md @@ -0,0 +1,57 @@ +--- +name: apply-house-style +description: Rewrite a paragraph to follow the house style recorded in the OKF wiki under okf/, citing the page each change came from. Use when a run asks you to apply house style to a passage. +--- + +# Apply house style from the wiki + +You are given one paragraph. Rewrite it so it follows the house style recorded +in the OKF wiki under `okf/`, and say which page each change came from. + +The wiki is **read-only** in this run and there is no source document to fall +back on — `okf/` is all you have, and it is the point. Answer from what the +pages actually say, not from what you already believe about house style. The +two often disagree, and here the wiki wins. + +## Procedure + +1. **Find the rulings.** Search `okf/` for the words and constructions used in + the paragraph. A ruling is usually a short section named after the word it + governs. Read the page before relying on it. +2. **Apply only what you found.** Make a change when a page supports it. Do not + make stylistic improvements the wiki does not ask for — an unsupported change + cannot be cited, and uncited changes are the failure this task looks for. +3. **Leave the rest alone.** Preserve the paragraph's meaning, facts and + figures. You are correcting style, not rewriting the argument. +4. **Report** in the format below. + +## Output format + +End your reply with a fenced `json` block, and put nothing after it: + +````text +```json +{ + "rewrite": "the full corrected paragraph", + "changes": [ + { + "before": "the original wording", + "after": "the corrected wording", + "ruling": "a short quote of what the page says", + "citation": "/part-2/6-confusables-and-cuttables-individual-rulings.md" + } + ] +} +``` +```` + +- `rewrite` — the whole paragraph, corrected. Not a diff, not a fragment. +- `changes` — one entry per change. An empty list is a valid answer if the + wiki genuinely supports no change. +- `ruling` — quote the page, briefly. Do not paraphrase it into something the + page does not say. +- `citation` — a **bundle-absolute** path to a page that exists, rooted at the + wiki root: `/part-2/….md`, not `okf/part-2/….md` and not a relative path. + +Cite the page you actually read. A citation to a page that does not contain the +ruling is worse than making no change at all: it presents a guess as sourced. diff --git a/scripts/test-house-style.sh b/scripts/test-house-style.sh new file mode 100755 index 0000000..770dc2e --- /dev/null +++ b/scripts/test-house-style.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Check that the compiled OKF wiki is USABLE, not merely well-formed. +# +# Usage: test-house-style.sh [cases.json] +# +# One Pi run per case, each given a badly written paragraph and nothing but the +# wiki: the `pi-consume` service mounts okf/ read-only and does not mount md/ at +# all, so the agent cannot consult the source documents. Each reply must end in +# a json block naming the changes and the page each came from; grade.py then +# checks the edit landed, the citation resolves, and the cited page really says +# what was claimed. +# +# Provider and model are passed explicitly rather than read from settings.json, +# so a test run states which model it exercised and leaves repo config alone. +# Override with PI_PROVIDER / PI_MODEL. +# +# Exit: 0 all cases passed, 1 one or more failed, 2 could not run. + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${repo_root}" + +cases_file="${1:-tests/house-style/cases.json}" +compose_file="pi/container/compose.yaml" +outdir="tests/house-style/out" +provider="${PI_PROVIDER:-litellm}" +model="${PI_MODEL:-gemini-3.1-pro-preview}" + +fail_infra() { + echo "error: $1" >&2 + exit 2 +} + +command -v docker >/dev/null 2>&1 || fail_infra "Docker CLI not found." +docker info >/dev/null 2>&1 || fail_infra "Docker does not appear to be running." +[[ -f "${cases_file}" ]] || fail_infra "Cases file not found: ${cases_file}" + +# The wiki is the artefact under test; without it the run is meaningless rather +# than failing. +compiled_pages="$(find okf -name '*.md' ! -name '.okflintrc.json' 2>/dev/null | wc -l | tr -d ' ')" +[[ "${compiled_pages}" -gt 0 ]] || fail_infra "No compiled wiki under okf/ — run a compile first." + +# Only the selected provider's key is needed. Checking here turns a missing +# credential into a clear message rather than an unauthenticated request that +# looks like a content failure. +case "${provider}" in +litellm) [[ -n "${LITELLM_API_KEY:-}" ]] || fail_infra "LITELLM_API_KEY is not set (provider=litellm)." ;; +openrouter) [[ -n "${OPENROUTER_API_KEY:-}" ]] || fail_infra "OPENROUTER_API_KEY is not set (provider=openrouter)." ;; +esac + +mkdir -p "${outdir}" +rm -f "${outdir}"/*.txt + +docker compose -f "${compose_file}" build pi-consume >/dev/null + +ids="$(python3 -c " +import json,sys +for c in json.load(open('${cases_file}'))['cases']: + print(c['id']) +")" + +echo "Consumption test: ${compiled_pages} wiki pages, provider=${provider}, model=${model}" +echo + +for id in ${ids}; do + paragraph="$(python3 -c " +import json +for c in json.load(open('${cases_file}'))['cases']: + if c['id'] == '${id}': + print(c['input']) + break +")" + echo " running ${id}" + # "${outdir}/${id}.txt" 2>&1 || true +done + +echo +python3 tests/house-style/grade.py okf "${cases_file}" "${outdir}" diff --git a/tests/house-style/cases.json b/tests/house-style/cases.json new file mode 100644 index 0000000..9030d50 --- /dev/null +++ b/tests/house-style/cases.json @@ -0,0 +1,101 @@ +{ + "_comment": [ + "Fixtures for the OKF consumption test. See", + "docs/superpowers/specs/2026-07-31-okf-consumption-test-design.md.", + "", + "Every 'grounding' string is copied verbatim from the compiled wiki. If a", + "case starts failing on 'grounded', check whether the compile changed the", + "wording before assuming the agent is at fault.", + "", + "Cases favour rulings a model cannot derive from general knowledge, several", + "of which invert the usual convention. That is what lets this test fail when", + "the agent answers from priors instead of reading the wiki.", + "", + "Matching is case-insensitive throughout.", + " expect_present - every string must appear in the rewrite", + " expect_absent - no string may appear in the rewrite", + " expect_any - at least one string must appear (open-ended fixes)" + ], + "cases": [ + { + "id": "percent-sign", + "input": "Revenues rose 12 per cent last year, and the board expects a further 4 per cent in 2026.", + "expect_present": ["12%"], + "expect_absent": ["per cent"], + "expect_any": [], + "grounding": "Use the sign % instead of per cent", + "expected_page": "/part-2/7-sweating-the-small-stuff-punctuation-mechanics-and-conventions.md", + "note": "Inverts the common convention: most guides spell it out, this one requires the sign." + }, + { + "id": "impact-as-verb", + "input": "The new tariff impacted margins across the region.", + "expect_present": [], + "expect_absent": ["impacted"], + "expect_any": [], + "grounding": "let one event impact another", + "expected_page": "/part-2/9-confusing-cousins-american-and-british-english.md", + "note": "House prohibition on verbing nouns; any valid replacement passes." + }, + { + "id": "due-to", + "input": "The launch was delayed due to a shortage of chips.", + "expect_present": [], + "expect_absent": ["due to"], + "expect_any": ["because of", "owing to", "on account of"], + "grounding": "Due to modifies a noun", + "expected_page": "/part-2/6-confusables-and-cuttables-individual-rulings.md", + "note": "Adverbial 'due to' is the disputed usage." + }, + { + "id": "decimate", + "input": "The frost decimated the entire orchard, leaving not a single tree alive.", + "expect_present": [], + "expect_absent": ["decimated"], + "expect_any": ["annihilated", "wiped out", "destroyed"], + "grounding": "destroy a significant proportion", + "expected_page": "/part-2/6-confusables-and-cuttables-individual-rulings.md", + "note": "The paragraph describes total destruction, which the ruling reserves other words for." + }, + { + "id": "fewer-than", + "input": "Less than seven firms bid for the contract.", + "expect_present": ["fewer"], + "expect_absent": ["less than seven"], + "expect_any": [], + "grounding": "Use fewer, not less, with individual items", + "expected_page": "/part-2/6-confusables-and-cuttables-individual-rulings.md", + "note": "Countable items take 'fewer'." + }, + { + "id": "hopefully", + "input": "Hopefully, the merger will close before the end of the quarter.", + "expect_present": [], + "expect_absent": ["hopefully"], + "expect_any": ["with luck", "if all goes well", "it is hoped"], + "grounding": "we do not write Hopefully", + "expected_page": "/part-2/6-confusables-and-cuttables-individual-rulings.md", + "note": "The wiki concedes this is not grammatically wrong and forbids it anyway - unguessable from priors." + }, + { + "id": "alibi", + "input": "The airline's alibi for the delay, a shortage of crew, convinced nobody.", + "expect_present": [], + "expect_absent": ["alibi"], + "expect_any": ["excuse", "explanation", "defence", "justification"], + "grounding": "not the same thing as a false explanation", + "expected_page": "/part-2/6-confusables-and-cuttables-individual-rulings.md", + "note": "An alibi is being elsewhere, not an excuse." + }, + { + "id": "compared-with", + "input": "Compared to last year, profits fell sharply while costs climbed.", + "expect_present": ["compared with"], + "expect_absent": ["compared to"], + "expect_any": [], + "grounding": "compared with is used to evaluate similarities", + "expected_page": "/part-2/6-confusables-and-cuttables-individual-rulings.md", + "note": "The sentence evaluates and highlights differences, so 'with'." + } + ] +} diff --git a/tests/house-style/grade.py b/tests/house-style/grade.py new file mode 100755 index 0000000..0c4eaae --- /dev/null +++ b/tests/house-style/grade.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Grade the OKF consumption test. + +Reads one agent transcript per case and applies three mechanical assertions. +There is no model in the loop here: every check is a string or filesystem +operation, so a run is reproducible and a failure is inspectable. + + applied every expect_present appears, no expect_absent appears, and at + least one expect_any appears when that list is non-empty + cited some changes[].citation resolves to a file inside the wiki + grounded one of those resolved pages actually contains the case's + grounding text + +`grounded` is the one that matters most. Without it an agent could produce the +right edit from memory, attach a plausible-looking path, and score full marks +without ever opening the wiki. + +Usage: grade.py +Exit: 0 all passed, 1 one or more failed, 2 could not run. +""" + +import json +import re +import sys +from pathlib import Path + +FENCE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL) + + +def extract_payload(text): + """Return the last fenced json block as a dict, or None.""" + blocks = FENCE.findall(text) + if not blocks: + return None + try: + payload = json.loads(blocks[-1]) + except json.JSONDecodeError: + return None + return payload if isinstance(payload, dict) else None + + +def resolve(wiki, citation): + """Map a bundle-absolute citation to a file on disk, or None.""" + if not isinstance(citation, str) or not citation.strip(): + return None + rel = citation.strip().lstrip("/") + # Tolerate an okf/ prefix even though the skill asks for bundle-absolute. + if rel.startswith(f"{wiki.name}/"): + rel = rel[len(wiki.name) + 1:] + path = wiki / rel + return path if path.is_file() else None + + +def grade(case, text, wiki): + """Return (passed, detail) for one case.""" + payload = extract_payload(text) + if payload is None: + return False, "no parseable json block in the reply" + + rewrite = payload.get("rewrite") + if not isinstance(rewrite, str) or not rewrite.strip(): + return False, "json block has no 'rewrite' string" + hay = rewrite.lower() + + missing = [s for s in case["expect_present"] if s.lower() not in hay] + present = [s for s in case["expect_absent"] if s.lower() in hay] + any_opts = case.get("expect_any") or [] + any_ok = (not any_opts) or any(s.lower() in hay for s in any_opts) + + if missing: + return False, f"applied: missing {missing}" + if present: + return False, f"applied: still contains {present}" + if not any_ok: + return False, f"applied: none of {any_opts} present" + + changes = payload.get("changes") + if not isinstance(changes, list) or not changes: + return False, "cited: no changes reported" + + resolved = [] + for ch in changes: + if isinstance(ch, dict): + hit = resolve(wiki, ch.get("citation", "")) + if hit is not None: + resolved.append(hit) + if not resolved: + cited = [c.get("citation") for c in changes if isinstance(c, dict)] + return False, f"cited: no citation resolves to a page ({cited})" + + needle = case["grounding"].lower() + for path in resolved: + if needle in path.read_text(errors="replace").lower(): + return True, f"cited {path.name}" + + names = ", ".join(p.name for p in resolved) + return False, f"grounded: no cited page contains {case['grounding']!r} (cited {names})" + + +def main(): + if len(sys.argv) != 4: + print(__doc__.strip().splitlines()[-2], file=sys.stderr) + return 2 + + wiki, cases_file, outdir = (Path(a) for a in sys.argv[1:4]) + if not wiki.is_dir(): + print(f"error: wiki directory not found: {wiki}", file=sys.stderr) + return 2 + if not cases_file.is_file(): + print(f"error: cases file not found: {cases_file}", file=sys.stderr) + return 2 + + cases = json.loads(cases_file.read_text())["cases"] + if not cases: + print("error: no cases defined", file=sys.stderr) + return 2 + + width = max(len(c["id"]) for c in cases) + results = [] + for case in cases: + transcript = outdir / f"{case['id']}.txt" + if not transcript.is_file(): + passed, detail = False, "no transcript (the run did not produce output)" + else: + passed, detail = grade(case, transcript.read_text(errors="replace"), wiki) + results.append(passed) + mark = "PASS" if passed else "FAIL" + print(f" {mark} {case['id']:<{width}} {detail}") + + ok = sum(results) + print(f"\n{ok}/{len(results)} cases passed") + return 0 if ok == len(results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) From f34f11f65eda5c5b2fd52180dc27e95b530659ad Mon Sep 17 00:00:00 2001 From: Maksym Prokopov Date: Fri, 31 Jul 2026 09:30:55 +0200 Subject: [PATCH 2/2] Accept a ruling that appears on more than one page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first real run failed impact-as-verb, and the agent was right while the fixture was wrong. It cited chapter 1, which says outright that "to impact" annoys enough readers that you should write "to have an impact on" — quoted it verbatim and applied exactly that fix. The fixture only accepted chapter 9's wording, where the same prohibition appears in a list of nouns not to verb. So `grounding` now takes a list and any one entry grounds the citation. This widens what counts as a correct source; it does not weaken the check that a source is required. Re-running the synthetic bad inputs confirms the hallucinated-citation case still fails. Also fixes the driver: compose interpolates every service in the file, so the sibling `pi` service's OPENROUTER_API_KEY fail-fast aborted a consumption run that never touches OpenRouter. The driver now supplies a placeholder purely to satisfy interpolation, after the real provider key check. --- scripts/test-house-style.sh | 7 +++++++ tests/house-style/cases.json | 7 +++++-- tests/house-style/grade.py | 14 ++++++++++---- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/scripts/test-house-style.sh b/scripts/test-house-style.sh index 770dc2e..aaa1ad5 100755 --- a/scripts/test-house-style.sh +++ b/scripts/test-house-style.sh @@ -52,6 +52,13 @@ esac mkdir -p "${outdir}" rm -f "${outdir}"/*.txt +# Compose interpolates every service in the file, not just the one being run, so +# the sibling `pi` service's OPENROUTER_API_KEY fail-fast would abort a +# consumption run that never touches OpenRouter. Supply a placeholder purely to +# satisfy interpolation; pi-consume itself takes the variable as optional, and +# nothing here sends it anywhere. +export OPENROUTER_API_KEY="${OPENROUTER_API_KEY:-unused-by-consumption-test}" + docker compose -f "${compose_file}" build pi-consume >/dev/null ids="$(python3 -c " diff --git a/tests/house-style/cases.json b/tests/house-style/cases.json index 9030d50..a4c2bcd 100644 --- a/tests/house-style/cases.json +++ b/tests/house-style/cases.json @@ -33,9 +33,12 @@ "expect_present": [], "expect_absent": ["impacted"], "expect_any": [], - "grounding": "let one event impact another", + "grounding": [ + "let one event impact another", + "no one is bothered by to impact" + ], "expected_page": "/part-2/9-confusing-cousins-american-and-british-english.md", - "note": "House prohibition on verbing nouns; any valid replacement passes." + "note": "House prohibition on verbing nouns; any valid replacement passes. The book rules on this twice — chapter 9 lists it among nouns not to verb, and chapter 1 names 'to impact' outright and prescribes 'to have an impact on'. Either page grounds the citation; an early run cited chapter 1 and was wrongly failed by a single-page grounding." }, { "id": "due-to", diff --git a/tests/house-style/grade.py b/tests/house-style/grade.py index 0c4eaae..7a8fbce 100755 --- a/tests/house-style/grade.py +++ b/tests/house-style/grade.py @@ -88,13 +88,19 @@ def grade(case, text, wiki): cited = [c.get("citation") for c in changes if isinstance(c, dict)] return False, f"cited: no citation resolves to a page ({cited})" - needle = case["grounding"].lower() + # A ruling may appear on more than one page — the book rules on verbing + # nouns in both chapter 1 and chapter 9, for instance — so `grounding` + # accepts a list and any one of them grounds the citation. + grounding = case["grounding"] + needles = [grounding] if isinstance(grounding, str) else list(grounding) for path in resolved: - if needle in path.read_text(errors="replace").lower(): - return True, f"cited {path.name}" + body = path.read_text(errors="replace").lower() + for needle in needles: + if needle.lower() in body: + return True, f"cited {path.name}" names = ", ".join(p.name for p in resolved) - return False, f"grounded: no cited page contains {case['grounding']!r} (cited {names})" + return False, f"grounded: no cited page contains any of {needles} (cited {names})" def main():