diff --git a/.gitignore b/.gitignore index 2813865..6c07e39 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,6 @@ md/* # okf output folder okf/* !okf/ -!okf/.okflintrc.json \ No newline at end of file +!okf/.okflintrc.json +# consumption-test transcripts (generated) +tests/house-style/out/ diff --git a/docs/superpowers/specs/2026-07-31-okf-consumption-test-design.md b/docs/superpowers/specs/2026-07-31-okf-consumption-test-design.md new file mode 100644 index 0000000..e1db1d8 --- /dev/null +++ b/docs/superpowers/specs/2026-07-31-okf-consumption-test-design.md @@ -0,0 +1,138 @@ +# OKF consumption test — design + +**Date:** 2026-07-31 +**Status:** approved, for implementation + +## Problem + +`md2okf` is verified end to end as a *producer*: a 382 KB book compiles into a +38-page OKF wiki that passes `okf-lint`. Nothing verifies the wiki is any +*use*. Completeness and well-formedness are not usefulness, and OKF's own goals +name consumption explicitly: + +> Inform how **consumption agents** should read and traverse it. +> — `SPEC.md`, Goals §2 + +So: can an agent holding only the compiled wiki apply the knowledge in it, and +can we tell the difference between it having done so and it having guessed? + +## The prior-knowledge problem + +This is the crux. A capable model already knows roughly what Economist house +style is. Hand it a badly written paragraph and a wiki, and a good rewrite +proves nothing — it may never have opened a page. + +Two mitigations, both required: + +1. **Arbitrary rulings.** Cases are built on house rulings a model cannot + derive from general knowledge, several of which invert the common + convention. The strongest is `%`: most style guides prefer spelling out + "per cent", and this one requires the sign. A model working from priors gets + it backwards. +2. **Verified citations.** Every change must cite the page it came from, and + grading checks the cited page *actually says so*. A plausible citation to a + page that does not support the claim fails. + +There is deliberately **no A/B control arm** (running each case again with the +wiki absent). It would be the cleanest evidence of contribution, but it doubles +the runs, and the counterintuitive rulings already give the test the ability to +fail when the wiki is unread. Recorded as a known limitation, not an oversight. + +## Isolation + +A second Compose service, `pi-consume`, reusing the existing image: + +| Mount | Compile run | Consumption run | +| --- | --- | --- | +| `md/` | read-only | **absent** | +| `okf/` | read-write | **read-only** | + +`md/` is unmounted so the agent cannot reach the source book — only the compiled +wiki. `okf/` is read-only because consumption must not mutate the artefact under +test; it also means the run needs no writable mount at all. + +The driver passes `--provider` and `--model` explicitly rather than editing +`settings.json`, so the test states which model it exercised and leaves repo +config untouched. + +## Output contract + +The agent writes nothing. Its final message is the result, captured from +`pi -p` stdout, and must end with a fenced `json` block: + +```json +{ + "rewrite": "the corrected paragraph", + "changes": [ + { + "before": "per cent", + "after": "%", + "ruling": "Use the sign % instead of per cent.", + "citation": "/part-2/7-sweating-the-small-stuff-punctuation-mechanics-and-conventions.md" + } + ] +} +``` + +The grader reads the **last** fenced `json` block, so surrounding prose is +harmless. A missing or unparseable block fails the case explicitly rather than +scoring zero silently. + +## Grading + +Three mechanical assertions per case. No LLM judge — every check is a string or +filesystem operation, so the result is deterministic and reproducible. + +1. **applied** — every `expect_present` string appears in `rewrite`, and every + `expect_absent` string does not. +2. **cited** — at least one `changes[].citation` resolves to a file that exists + in the bundle. +3. **grounded** — that cited file contains the case's `grounding` text. + +A case passes only when all three hold. Assertion 3 is what makes hallucinated +citations fail: getting the edit right by luck and inventing a source still +fails the case. + +## Cases + +All eight are verified present in the compiled wiki. `ch6` is +`/part-2/6-confusables-and-cuttables-individual-rulings.md`. + +| # | Ruling | Page | Discriminates because | +| --- | --- | --- | --- | +| 1 | `%` not "per cent" | ch7 | inverts the usual convention | +| 2 | do not verb "impact" | ch9 | house prohibition, not grammar | +| 3 | "due to" modifies nouns only | ch6 | prefers *because of* / *owing to* | +| 4 | "decimate" = a significant proportion | ch6 | not total destruction | +| 5 | "fewer" for countables | ch6 | common error, explicit ruling | +| 6 | no sentence-initial "Hopefully," | ch6 | explicitly *not* a grammar rule | +| 7 | "alibi" ≠ excuse | ch6 | semantic ruling | +| 8 | "compared with" for evaluation | ch6 | fine distinction against *compared to* | + +## Failure taxonomy + +Infrastructure failure must never read as "the wiki is bad". The driver exits: + +- `0` — all cases passed +- `1` — at least one case failed on its assertions (a real result) +- `2` — infrastructure trouble: missing key, gateway error, no wiki, no cases + +## Components + +| Path | Purpose | +| --- | --- | +| `pi/{container,sandbox}/…/skills/apply-house-style/SKILL.md` | the consumption task, one copy per runtime, aligned by hand | +| `tests/house-style/cases.yaml` | fixtures: input, assertions, expected page, grounding | +| `tests/house-style/grade.py` | the three assertions, table output, exit code | +| `scripts/test-house-style.sh` | driver: one Pi run per case, then grade | +| `pi/container/compose.yaml` | adds the `pi-consume` service | + +## Non-goals + +- **Navigability.** The container ships `ripgrep` and `fd`, so the agent will + likely grep rather than traverse `index.md` files. This test says nothing + about whether the index structure works. +- **Prose quality.** Only the specific rulings are graded, not whether the + rewrite reads well. +- **The sandbox runtime.** Implemented for parity, exercised on the container + runtime only, matching how every other result in this repo was obtained. diff --git a/pi/container/agent/skills/apply-house-style/SKILL.md b/pi/container/agent/skills/apply-house-style/SKILL.md new file mode 100644 index 0000000..1233494 --- /dev/null +++ b/pi/container/agent/skills/apply-house-style/SKILL.md @@ -0,0 +1,57 @@ +--- +name: apply-house-style +description: Rewrite a paragraph to follow the house style recorded in the OKF wiki under okf/, citing the page each change came from. Use when a run asks you to apply house style to a passage. +--- + +# Apply house style from the wiki + +You are given one paragraph. Rewrite it so it follows the house style recorded +in the OKF wiki under `okf/`, and say which page each change came from. + +The wiki is **read-only** in this run and there is no source document to fall +back on — `okf/` is all you have, and it is the point. Answer from what the +pages actually say, not from what you already believe about house style. The +two often disagree, and here the wiki wins. + +## Procedure + +1. **Find the rulings.** Search `okf/` for the words and constructions used in + the paragraph. A ruling is usually a short section named after the word it + governs. Read the page before relying on it. +2. **Apply only what you found.** Make a change when a page supports it. Do not + make stylistic improvements the wiki does not ask for — an unsupported change + cannot be cited, and uncited changes are the failure this task looks for. +3. **Leave the rest alone.** Preserve the paragraph's meaning, facts and + figures. You are correcting style, not rewriting the argument. +4. **Report** in the format below. + +## Output format + +End your reply with a fenced `json` block, and put nothing after it: + +````text +```json +{ + "rewrite": "the full corrected paragraph", + "changes": [ + { + "before": "the original wording", + "after": "the corrected wording", + "ruling": "a short quote of what the page says", + "citation": "/part-2/6-confusables-and-cuttables-individual-rulings.md" + } + ] +} +``` +```` + +- `rewrite` — the whole paragraph, corrected. Not a diff, not a fragment. +- `changes` — one entry per change. An empty list is a valid answer if the + wiki genuinely supports no change. +- `ruling` — quote the page, briefly. Do not paraphrase it into something the + page does not say. +- `citation` — a **bundle-absolute** path to a page that exists, rooted at the + wiki root: `/part-2/….md`, not `okf/part-2/….md` and not a relative path. + +Cite the page you actually read. A citation to a page that does not contain the +ruling is worse than making no change at all: it presents a guess as sourced. diff --git a/pi/container/compose.yaml b/pi/container/compose.yaml index d1c64da..7259efb 100644 --- a/pi/container/compose.yaml +++ b/pi/container/compose.yaml @@ -56,5 +56,53 @@ services: - ../../md:/workspace/md:ro - ../../okf:/workspace/okf:rw + # Consumption runtime: reads the compiled wiki, never writes it. Used by + # scripts/test-house-style.sh to check the wiki is usable, not merely + # well-formed. Two differences from `pi` above carry the whole point: + # + # - md/ is NOT mounted, so the agent cannot reach the source documents and + # must answer from okf/ alone. Mount it and the test proves nothing. + # - okf/ is read-only, because the artefact under test must survive the test + # unchanged. Nothing is written, so no writable mount is needed; the + # agent's answer is its stdout. + # + # Neither key is required here — the driver selects a provider with + # --provider/--model, and only that provider's key has to be set. + pi-consume: + build: + context: ../.. + dockerfile: pi/container/Dockerfile + image: pi-container + + environment: + HOME: /home/node + OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-} + LITELLM_API_KEY: ${LITELLM_API_KEY:-} + + network_mode: bridge + cap_drop: + - ALL + security_opt: + - no-new-privileges + read_only: true + user: "1000:1000" + + pids_limit: 512 + mem_limit: 4g + cpus: 2 + + tmpfs: + - /tmp + - /run + - /home/node + + volumes: + - pi-agent-home:/home/node/.pi/agent + - ./agent/settings.json:/home/node/.pi/agent/settings.json:ro + - ./agent/models.json:/home/node/.pi/agent/models.json:ro + - ./agent/AGENTS.md:/home/node/.pi/agent/AGENTS.md:ro + - ./agent/skills:/home/node/.pi/agent/skills:ro + - ../../okf:/workspace/okf:ro + volumes: pi-agent-home: diff --git a/pi/sandbox/files/home/.pi/agent/skills/apply-house-style/SKILL.md b/pi/sandbox/files/home/.pi/agent/skills/apply-house-style/SKILL.md new file mode 100644 index 0000000..1233494 --- /dev/null +++ b/pi/sandbox/files/home/.pi/agent/skills/apply-house-style/SKILL.md @@ -0,0 +1,57 @@ +--- +name: apply-house-style +description: Rewrite a paragraph to follow the house style recorded in the OKF wiki under okf/, citing the page each change came from. Use when a run asks you to apply house style to a passage. +--- + +# Apply house style from the wiki + +You are given one paragraph. Rewrite it so it follows the house style recorded +in the OKF wiki under `okf/`, and say which page each change came from. + +The wiki is **read-only** in this run and there is no source document to fall +back on — `okf/` is all you have, and it is the point. Answer from what the +pages actually say, not from what you already believe about house style. The +two often disagree, and here the wiki wins. + +## Procedure + +1. **Find the rulings.** Search `okf/` for the words and constructions used in + the paragraph. A ruling is usually a short section named after the word it + governs. Read the page before relying on it. +2. **Apply only what you found.** Make a change when a page supports it. Do not + make stylistic improvements the wiki does not ask for — an unsupported change + cannot be cited, and uncited changes are the failure this task looks for. +3. **Leave the rest alone.** Preserve the paragraph's meaning, facts and + figures. You are correcting style, not rewriting the argument. +4. **Report** in the format below. + +## Output format + +End your reply with a fenced `json` block, and put nothing after it: + +````text +```json +{ + "rewrite": "the full corrected paragraph", + "changes": [ + { + "before": "the original wording", + "after": "the corrected wording", + "ruling": "a short quote of what the page says", + "citation": "/part-2/6-confusables-and-cuttables-individual-rulings.md" + } + ] +} +``` +```` + +- `rewrite` — the whole paragraph, corrected. Not a diff, not a fragment. +- `changes` — one entry per change. An empty list is a valid answer if the + wiki genuinely supports no change. +- `ruling` — quote the page, briefly. Do not paraphrase it into something the + page does not say. +- `citation` — a **bundle-absolute** path to a page that exists, rooted at the + wiki root: `/part-2/….md`, not `okf/part-2/….md` and not a relative path. + +Cite the page you actually read. A citation to a page that does not contain the +ruling is worse than making no change at all: it presents a guess as sourced. diff --git a/scripts/test-house-style.sh b/scripts/test-house-style.sh new file mode 100755 index 0000000..aaa1ad5 --- /dev/null +++ b/scripts/test-house-style.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Check that the compiled OKF wiki is USABLE, not merely well-formed. +# +# Usage: test-house-style.sh [cases.json] +# +# One Pi run per case, each given a badly written paragraph and nothing but the +# wiki: the `pi-consume` service mounts okf/ read-only and does not mount md/ at +# all, so the agent cannot consult the source documents. Each reply must end in +# a json block naming the changes and the page each came from; grade.py then +# checks the edit landed, the citation resolves, and the cited page really says +# what was claimed. +# +# Provider and model are passed explicitly rather than read from settings.json, +# so a test run states which model it exercised and leaves repo config alone. +# Override with PI_PROVIDER / PI_MODEL. +# +# Exit: 0 all cases passed, 1 one or more failed, 2 could not run. + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${repo_root}" + +cases_file="${1:-tests/house-style/cases.json}" +compose_file="pi/container/compose.yaml" +outdir="tests/house-style/out" +provider="${PI_PROVIDER:-litellm}" +model="${PI_MODEL:-gemini-3.1-pro-preview}" + +fail_infra() { + echo "error: $1" >&2 + exit 2 +} + +command -v docker >/dev/null 2>&1 || fail_infra "Docker CLI not found." +docker info >/dev/null 2>&1 || fail_infra "Docker does not appear to be running." +[[ -f "${cases_file}" ]] || fail_infra "Cases file not found: ${cases_file}" + +# The wiki is the artefact under test; without it the run is meaningless rather +# than failing. +compiled_pages="$(find okf -name '*.md' ! -name '.okflintrc.json' 2>/dev/null | wc -l | tr -d ' ')" +[[ "${compiled_pages}" -gt 0 ]] || fail_infra "No compiled wiki under okf/ — run a compile first." + +# Only the selected provider's key is needed. Checking here turns a missing +# credential into a clear message rather than an unauthenticated request that +# looks like a content failure. +case "${provider}" in +litellm) [[ -n "${LITELLM_API_KEY:-}" ]] || fail_infra "LITELLM_API_KEY is not set (provider=litellm)." ;; +openrouter) [[ -n "${OPENROUTER_API_KEY:-}" ]] || fail_infra "OPENROUTER_API_KEY is not set (provider=openrouter)." ;; +esac + +mkdir -p "${outdir}" +rm -f "${outdir}"/*.txt + +# Compose interpolates every service in the file, not just the one being run, so +# the sibling `pi` service's OPENROUTER_API_KEY fail-fast would abort a +# consumption run that never touches OpenRouter. Supply a placeholder purely to +# satisfy interpolation; pi-consume itself takes the variable as optional, and +# nothing here sends it anywhere. +export OPENROUTER_API_KEY="${OPENROUTER_API_KEY:-unused-by-consumption-test}" + +docker compose -f "${compose_file}" build pi-consume >/dev/null + +ids="$(python3 -c " +import json,sys +for c in json.load(open('${cases_file}'))['cases']: + print(c['id']) +")" + +echo "Consumption test: ${compiled_pages} wiki pages, provider=${provider}, model=${model}" +echo + +for id in ${ids}; do + paragraph="$(python3 -c " +import json +for c in json.load(open('${cases_file}'))['cases']: + if c['id'] == '${id}': + print(c['input']) + break +")" + echo " running ${id}" + # "${outdir}/${id}.txt" 2>&1 || true +done + +echo +python3 tests/house-style/grade.py okf "${cases_file}" "${outdir}" diff --git a/tests/house-style/cases.json b/tests/house-style/cases.json new file mode 100644 index 0000000..a4c2bcd --- /dev/null +++ b/tests/house-style/cases.json @@ -0,0 +1,104 @@ +{ + "_comment": [ + "Fixtures for the OKF consumption test. See", + "docs/superpowers/specs/2026-07-31-okf-consumption-test-design.md.", + "", + "Every 'grounding' string is copied verbatim from the compiled wiki. If a", + "case starts failing on 'grounded', check whether the compile changed the", + "wording before assuming the agent is at fault.", + "", + "Cases favour rulings a model cannot derive from general knowledge, several", + "of which invert the usual convention. That is what lets this test fail when", + "the agent answers from priors instead of reading the wiki.", + "", + "Matching is case-insensitive throughout.", + " expect_present - every string must appear in the rewrite", + " expect_absent - no string may appear in the rewrite", + " expect_any - at least one string must appear (open-ended fixes)" + ], + "cases": [ + { + "id": "percent-sign", + "input": "Revenues rose 12 per cent last year, and the board expects a further 4 per cent in 2026.", + "expect_present": ["12%"], + "expect_absent": ["per cent"], + "expect_any": [], + "grounding": "Use the sign % instead of per cent", + "expected_page": "/part-2/7-sweating-the-small-stuff-punctuation-mechanics-and-conventions.md", + "note": "Inverts the common convention: most guides spell it out, this one requires the sign." + }, + { + "id": "impact-as-verb", + "input": "The new tariff impacted margins across the region.", + "expect_present": [], + "expect_absent": ["impacted"], + "expect_any": [], + "grounding": [ + "let one event impact another", + "no one is bothered by to impact" + ], + "expected_page": "/part-2/9-confusing-cousins-american-and-british-english.md", + "note": "House prohibition on verbing nouns; any valid replacement passes. The book rules on this twice — chapter 9 lists it among nouns not to verb, and chapter 1 names 'to impact' outright and prescribes 'to have an impact on'. Either page grounds the citation; an early run cited chapter 1 and was wrongly failed by a single-page grounding." + }, + { + "id": "due-to", + "input": "The launch was delayed due to a shortage of chips.", + "expect_present": [], + "expect_absent": ["due to"], + "expect_any": ["because of", "owing to", "on account of"], + "grounding": "Due to modifies a noun", + "expected_page": "/part-2/6-confusables-and-cuttables-individual-rulings.md", + "note": "Adverbial 'due to' is the disputed usage." + }, + { + "id": "decimate", + "input": "The frost decimated the entire orchard, leaving not a single tree alive.", + "expect_present": [], + "expect_absent": ["decimated"], + "expect_any": ["annihilated", "wiped out", "destroyed"], + "grounding": "destroy a significant proportion", + "expected_page": "/part-2/6-confusables-and-cuttables-individual-rulings.md", + "note": "The paragraph describes total destruction, which the ruling reserves other words for." + }, + { + "id": "fewer-than", + "input": "Less than seven firms bid for the contract.", + "expect_present": ["fewer"], + "expect_absent": ["less than seven"], + "expect_any": [], + "grounding": "Use fewer, not less, with individual items", + "expected_page": "/part-2/6-confusables-and-cuttables-individual-rulings.md", + "note": "Countable items take 'fewer'." + }, + { + "id": "hopefully", + "input": "Hopefully, the merger will close before the end of the quarter.", + "expect_present": [], + "expect_absent": ["hopefully"], + "expect_any": ["with luck", "if all goes well", "it is hoped"], + "grounding": "we do not write Hopefully", + "expected_page": "/part-2/6-confusables-and-cuttables-individual-rulings.md", + "note": "The wiki concedes this is not grammatically wrong and forbids it anyway - unguessable from priors." + }, + { + "id": "alibi", + "input": "The airline's alibi for the delay, a shortage of crew, convinced nobody.", + "expect_present": [], + "expect_absent": ["alibi"], + "expect_any": ["excuse", "explanation", "defence", "justification"], + "grounding": "not the same thing as a false explanation", + "expected_page": "/part-2/6-confusables-and-cuttables-individual-rulings.md", + "note": "An alibi is being elsewhere, not an excuse." + }, + { + "id": "compared-with", + "input": "Compared to last year, profits fell sharply while costs climbed.", + "expect_present": ["compared with"], + "expect_absent": ["compared to"], + "expect_any": [], + "grounding": "compared with is used to evaluate similarities", + "expected_page": "/part-2/6-confusables-and-cuttables-individual-rulings.md", + "note": "The sentence evaluates and highlights differences, so 'with'." + } + ] +} diff --git a/tests/house-style/grade.py b/tests/house-style/grade.py new file mode 100755 index 0000000..7a8fbce --- /dev/null +++ b/tests/house-style/grade.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Grade the OKF consumption test. + +Reads one agent transcript per case and applies three mechanical assertions. +There is no model in the loop here: every check is a string or filesystem +operation, so a run is reproducible and a failure is inspectable. + + applied every expect_present appears, no expect_absent appears, and at + least one expect_any appears when that list is non-empty + cited some changes[].citation resolves to a file inside the wiki + grounded one of those resolved pages actually contains the case's + grounding text + +`grounded` is the one that matters most. Without it an agent could produce the +right edit from memory, attach a plausible-looking path, and score full marks +without ever opening the wiki. + +Usage: grade.py +Exit: 0 all passed, 1 one or more failed, 2 could not run. +""" + +import json +import re +import sys +from pathlib import Path + +FENCE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL) + + +def extract_payload(text): + """Return the last fenced json block as a dict, or None.""" + blocks = FENCE.findall(text) + if not blocks: + return None + try: + payload = json.loads(blocks[-1]) + except json.JSONDecodeError: + return None + return payload if isinstance(payload, dict) else None + + +def resolve(wiki, citation): + """Map a bundle-absolute citation to a file on disk, or None.""" + if not isinstance(citation, str) or not citation.strip(): + return None + rel = citation.strip().lstrip("/") + # Tolerate an okf/ prefix even though the skill asks for bundle-absolute. + if rel.startswith(f"{wiki.name}/"): + rel = rel[len(wiki.name) + 1:] + path = wiki / rel + return path if path.is_file() else None + + +def grade(case, text, wiki): + """Return (passed, detail) for one case.""" + payload = extract_payload(text) + if payload is None: + return False, "no parseable json block in the reply" + + rewrite = payload.get("rewrite") + if not isinstance(rewrite, str) or not rewrite.strip(): + return False, "json block has no 'rewrite' string" + hay = rewrite.lower() + + missing = [s for s in case["expect_present"] if s.lower() not in hay] + present = [s for s in case["expect_absent"] if s.lower() in hay] + any_opts = case.get("expect_any") or [] + any_ok = (not any_opts) or any(s.lower() in hay for s in any_opts) + + if missing: + return False, f"applied: missing {missing}" + if present: + return False, f"applied: still contains {present}" + if not any_ok: + return False, f"applied: none of {any_opts} present" + + changes = payload.get("changes") + if not isinstance(changes, list) or not changes: + return False, "cited: no changes reported" + + resolved = [] + for ch in changes: + if isinstance(ch, dict): + hit = resolve(wiki, ch.get("citation", "")) + if hit is not None: + resolved.append(hit) + if not resolved: + cited = [c.get("citation") for c in changes if isinstance(c, dict)] + return False, f"cited: no citation resolves to a page ({cited})" + + # A ruling may appear on more than one page — the book rules on verbing + # nouns in both chapter 1 and chapter 9, for instance — so `grounding` + # accepts a list and any one of them grounds the citation. + grounding = case["grounding"] + needles = [grounding] if isinstance(grounding, str) else list(grounding) + for path in resolved: + body = path.read_text(errors="replace").lower() + for needle in needles: + if needle.lower() in body: + return True, f"cited {path.name}" + + names = ", ".join(p.name for p in resolved) + return False, f"grounded: no cited page contains any of {needles} (cited {names})" + + +def main(): + if len(sys.argv) != 4: + print(__doc__.strip().splitlines()[-2], file=sys.stderr) + return 2 + + wiki, cases_file, outdir = (Path(a) for a in sys.argv[1:4]) + if not wiki.is_dir(): + print(f"error: wiki directory not found: {wiki}", file=sys.stderr) + return 2 + if not cases_file.is_file(): + print(f"error: cases file not found: {cases_file}", file=sys.stderr) + return 2 + + cases = json.loads(cases_file.read_text())["cases"] + if not cases: + print("error: no cases defined", file=sys.stderr) + return 2 + + width = max(len(c["id"]) for c in cases) + results = [] + for case in cases: + transcript = outdir / f"{case['id']}.txt" + if not transcript.is_file(): + passed, detail = False, "no transcript (the run did not produce output)" + else: + passed, detail = grade(case, transcript.read_text(errors="replace"), wiki) + results.append(passed) + mark = "PASS" if passed else "FAIL" + print(f" {mark} {case['id']:<{width}} {detail}") + + ok = sum(results) + print(f"\n{ok}/{len(results)} cases passed") + return 0 if ok == len(results) else 1 + + +if __name__ == "__main__": + sys.exit(main())