diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 4c5f5d6..cd299ef 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -15,6 +15,13 @@ "name": "tastytrade", "source": "./plugins/tastytrade", "description": "Inspect TastyTrade brokerage accounts, positions, market data, option chains, and transactions from Claude Code. Order placement is gated off by default." + }, + { + "name": "prove-the-test-fails", + "source": "./", + "strict": false, + "skills": ["./skills/prove-the-test-fails"], + "description": "Break the code under test to confirm a test can actually fail, and fails for the right reason, before trusting it." } ] } diff --git a/.github/workflows/skill-prove-the-test-fails.yml b/.github/workflows/skill-prove-the-test-fails.yml new file mode 100644 index 0000000..0c9aaa9 --- /dev/null +++ b/.github/workflows/skill-prove-the-test-fails.yml @@ -0,0 +1,59 @@ +name: skill prove-the-test-fails + +# One workflow per skill, matching the per-component paths filters used for the +# MCP servers, so a change to one skill never runs another's checks. +# +# marketplace.json is in the filter because it is the shared file: an entry +# edited by another skill's pull request can silently stop this one loading. +on: + pull_request: + paths: + - "skills/prove-the-test-fails/**" + - ".claude-plugin/marketplace.json" + - ".github/workflows/skill-prove-the-test-fails.yml" + push: + branches: [main] + paths: + - "skills/prove-the-test-fails/**" + - ".claude-plugin/marketplace.json" + - ".github/workflows/skill-prove-the-test-fails.yml" + +defaults: + run: + working-directory: skills/prove-the-test-fails + +jobs: + wiring: + # Packaging, marketplace entry, and the fixture's honesty. No credentials. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + - run: make check + - run: npm install -g @anthropic-ai/claude-code + - name: Validate the marketplace manifest + working-directory: . + run: claude plugin validate . --strict + + evals: + # Behavioural: does the skill lead to breaking the code and reading the + # result. Costs Anthropic tokens, so it skips without a key (fork PRs get + # no secrets). + runs-on: ubuntu-latest + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + - name: Run the behavioural eval + run: | + if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + echo "ANTHROPIC_API_KEY absent (fork PR?); skipping the behavioural eval." + exit 0 + fi + npm install -g @anthropic-ai/claude-code + make evals diff --git a/README.md b/README.md index e03955b..621ef30 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,18 @@ Formerly `walkerhughes/mcps`, back when it only held MCP servers. They follow Honeycomb's [MCP, easy as 1-2-3](https://www.honeycomb.io/blog/mcp-easy-as-1-2-3) guidance: a few curated tools built around real questions rather than raw API endpoints, responses shaped for a model instead of a UI, and typed schemas that steer the model toward valid calls. +## Skills + +Skills live in top-level `skills/`, one directory each, and ship through the marketplace as their own installable entries. + +| Skill | What it does | +|-------|--------------| +| [`prove-the-test-fails`](skills/prove-the-test-fails/) | Breaks the code under test to confirm a test can actually fail, and fails for the right reason, before trusting it. | + +A skill's marketplace entry sets `source: "./"` with a `skills` path pointing at its own directory, so several skills share the one top-level folder without loading each other, and `strict: false` because the repository root has no `plugin.json` to be the authority. Entries deliberately carry no `version`: Claude Code then resolves the version from the commit SHA, so every change reaches installed copies without a manual bump, and the stale-cache trap described below does not apply. + +Each skill directory carries its own checks, run from that directory: `make check` for packaging and wiring, which needs no credentials, and `make evals` for behaviour, which costs tokens. CI runs both per skill through `paths` filters, as it does for the servers. + ## Install a plugin Run these as two separate commands, not as one paste: the first opens a prompt that expects only the `owner/repo`. @@ -44,11 +56,13 @@ Plugins require [`uv`](https://docs.astral.sh/uv/) on your PATH. The first launc ``` claude/ ├── .claude-plugin/ # marketplace manifest -└── plugins/ - ├── harbor-hub/ - └── tastytrade/ +├── plugins/ +│ ├── harbor-hub/ +│ └── tastytrade/ +└── skills/ + └── prove-the-test-fails/ ``` -Plugins live under `plugins/`, one directory each, named for the platform they talk to rather than for being an MCP server. Skills and other components get their own top-level directories as they arrive. +Plugins live under `plugins/`, one directory each, named for the platform they talk to rather than for being an MCP server. Skills live under `skills/`, named for the practice they encode. Other components get their own top-level directories as they arrive. Each plugin directory is self-contained: its own `README.md` covers install, credentials, tests, and tools. CI runs per subdirectory via `paths` filters, so a change to one never runs another's suite. diff --git a/skills/prove-the-test-fails/Makefile b/skills/prove-the-test-fails/Makefile new file mode 100644 index 0000000..9f5b9a4 --- /dev/null +++ b/skills/prove-the-test-fails/Makefile @@ -0,0 +1,10 @@ +.PHONY: check evals + +# Packaging, wiring, and the fixture's honesty. No API key, no cost. +check: + python3 evals/check_wiring.py + ./evals/check_fixture.sh + +# Behavioural eval. Needs Claude Code on PATH and working credentials. +evals: + ./evals/run_eval.sh diff --git a/skills/prove-the-test-fails/SKILL.md b/skills/prove-the-test-fails/SKILL.md new file mode 100644 index 0000000..1b6349b --- /dev/null +++ b/skills/prove-the-test-fails/SKILL.md @@ -0,0 +1,52 @@ +--- +name: prove-the-test-fails +description: Break the code under test to confirm a test can actually fail, and fails for the right reason, before trusting it. Use after writing or changing a test, before citing a green run as evidence that something works, when a test has never been seen red, when a suite is said to enforce a contract or invariant, or when an assertion could be satisfied by an empty or degenerate result. +--- + +# Prove the test fails + +A test is not evidence until it has been observed failing for the reason it exists. A green +run says the assertions did not raise; it does not say they could. So break the thing the +test guards, and watch what happens. + +Two things follow. Break at the seam the test claims to guard rather than wherever a break +is easy, because a red run the test did not cause proves nothing about that test. And +surviving a mutation does not make a test decorative, since it may assert something the +mutation left true. The question is never whether a test survived but whether it can fail +for the reason it exists, so re-aim at its own claim before calling it decorative. + +## The loop + +1. **Pick the mutation.** The smallest change to the code under test that should trip this + test, applied where the test says it is looking. If it claims two implementations agree, + break one of them. Not a shared import, a fixture, or a build setting: those fail + everything and say nothing about the test in front of you. +2. **Run the suite and read the output**, not the exit code. +3. **Confirm the failure is the right one.** The test under scrutiny is among the failures, + and its message names what that test protects. A run that dies in an import or a fixture + has verified nothing. +4. **Confirm the blast radius.** Where one test body runs over several inputs or + implementations, breaking one must fail its own cases and leave the rest green. The + wrong radius means the test measures something other than its name. +5. **Revert and confirm the suite is green.** Always, and before anything else. A mutation + left behind is a broken repository. + +One mutation at a time, so the failures have one cause. Independent seams are therefore +independent runs, and the loop fans out: give each seam its own working copy, run the loop +there, and collect which cases each break turned red. + +## An assertion that cannot fail + +Assertions about shape rather than content go vacuous quietly. A contract test comparing +the result types of two implementations, on an input that matches nothing, reduces to +comparing two empty sets: it passes, it names a contract, and it guards nothing. Sorted key +lists, lengths, and non-null checks fail the same way. Suspect those first, along with any +green never yet contradicted. + +## Evidence, not a count + +> Made the cache's read return nothing unconditionally. All 8 cases in the `redis` group +> failed, the 8 `memory` cases passed, nothing else moved. Reverted, suite green. + +The mutation, the cases that failed, the cases that did not, and the confirmed revert are +evidence. "50 tests pass" is not. diff --git a/skills/prove-the-test-fails/evals/README.md b/skills/prove-the-test-fails/evals/README.md new file mode 100644 index 0000000..9c3b584 --- /dev/null +++ b/skills/prove-the-test-fails/evals/README.md @@ -0,0 +1,73 @@ +# prove-the-test-fails evals + +Two checks over one fixture. The fixture is a small route matcher with a suite that is +green and a contract test that cannot fail, reproducing a defect shape found in a real +codebase: an assertion comparing sets of result types, on an input that matches nothing, +so it reduces to `set() == set()` whatever the implementations do. + +``` +fixture/ +├── src/routing.py two implementations of one matcher over one route table +└── tests/test_contract.py five passing tests, one of them decorative +``` + +Python because the checks need some real codebase to run, not because the skill is about +Python. A fixture per ecosystem would cost a lot and measure the same method. + +The fixture also carries the two cases the skill warns about. Stubbing `regex_router` to +return nothing fails only the `[regex]` case of `test_matches_a_route_with_a_parameter` +and leaves the `[segment]` case green, which is the blast radius a correct mutation +produces. And `test_empty_path_matches_nothing[regex]` survives that same mutation +legitimately, because asserting emptiness is satisfied by an empty implementation. + +## The eval + +```bash +./run_eval.sh [--model sonnet] # costs LLM tokens +./run_eval.sh --control # same task, skill absent +``` + +Copies the fixture into a scratch git repository, drops `SKILL.md` in as a project skill, +and asks a headless Claude Code run whether the suite actually guards the rule it claims +to. Grading is on behaviour rather than prose: + +| Check | Why | +| --- | --- | +| First line of `VERDICT.txt` is `UNGUARDED` | Only knowable by breaking an implementation and watching the contract test stay green | +| `src`, `tests`, and `pyproject.toml` match the starting commit | Any mutation was reverted | +| The suite is green again | The repository was left working | + +The skill is never named in the prompt, so an automatic load is also a test of the +`description` field. + +**What this measures.** It is a regression guard on the guidance: an edit to `SKILL.md` +that drops the revert step, or that stops the skill loading on this kind of question, +turns it red. It is not evidence of uplift. In the runs on record the control passes too, +so a current model reaches the same answer on this fixture unprompted. Raising the +fixture's difficulty until the control fails is the way to turn this into an uplift +measurement, and until that happens the eval should not be described as one. + +## The fixture check + +```bash +./check_fixture.sh # no agent, no LLM, runs in CI +``` + +The eval only asks a real question while the fixture's contract test genuinely cannot +fail. This applies the mutation an agent is expected to find and asserts the exact shape +of the run that follows: the contract test survives, the `[regex]` parameter case dies, +the `[segment]` case lives, and the empty-path case survives. Repairing the fixture's +assertion turns this red with a message saying the eval no longer poses its question. It +works on a copy, so the checked-in fixture is never mutated. + +## Why not a Harbor task + +The MCP plugins in this repo gate on [Harbor](https://www.harborframework.com) tasks, +which is the right shape there: an MCP server has to be exercised as a server, in a +container, against a live hub. A skill is a markdown file that has to be present in the +agent's own skill directory, and the version under test is the one in the branch. The +Harbor tasks here install their subject from GitHub's default branch, so a task would +gate on the published skill rather than the change under review. Containerising also +brings a hub key and Modal for no gain, since nothing here talks to a hub. A local +headless run keeps the same three phases, fixture then agent then grade, without any of +that. diff --git a/skills/prove-the-test-fails/evals/check_fixture.sh b/skills/prove-the-test-fails/evals/check_fixture.sh new file mode 100755 index 0000000..77bc23a --- /dev/null +++ b/skills/prove-the-test-fails/evals/check_fixture.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Proves the eval fixture is still a trap. No agent, no LLM, no network beyond +# fetching pytest. +# +# The agentic eval is only meaningful while `test_routers_satisfy_the_same_contract` +# genuinely cannot fail. This applies the mutation the eval expects an agent to find, +# stubbing `regex_router` to return nothing, and asserts the exact shape of the +# resulting run: the contract test survives, the regex case of the parameter test dies, +# the segment case lives, and the empty-path test survives for its own good reason. If +# someone repairs the fixture's assertion, this goes red and says so. +# +# Runs on a copy: the checked-in fixture is never mutated. +set -euo pipefail + +EVALS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PYTEST=(uv run --no-project --with pytest pytest -v --tb=no -p no:cacheprovider) + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +cp -R "$EVALS_DIR/fixture/." "$work/" + +report="$work/report.txt" + +echo "==> baseline: the fixture suite must be green" +(cd "$work" && "${PYTEST[@]}") > "$report" 2>&1 || { + cat "$report" + echo "FAIL: the untouched fixture suite is not green." >&2 + exit 1 +} + +echo "==> mutation: stub regex_router to return nothing" +(cd "$work" && python3 - <<'PY' +import pathlib + +source = pathlib.Path("src/routing.py") +text = source.read_text() +anchor = ' """Compiled patterns: each route becomes a regex the whole path must match."""\n' +if anchor not in text: + raise SystemExit("FAIL: regex_router no longer looks the way this check expects.") +source.write_text(text.replace(anchor, anchor + " return []\n", 1)) +PY +) + +echo "==> the mutated suite must fail in one specific shape" +(cd "$work" && "${PYTEST[@]}") > "$report" 2>&1 || true + +status=0 +expect() { + local node=$1 want=$2 why=$3 + if grep -qF "::${node} ${want}" "$report"; then + echo " ok ${node} ${want}" + else + echo " FAIL ${node} expected ${want}: ${why}" >&2 + status=1 + fi +} + +# The trap itself: the one test that claims to guard the shared contract does not +# notice that an implementation stopped returning anything. +expect "test_routers_satisfy_the_same_contract" PASSED \ + "the fixture's contract test can fail now, so the eval no longer poses the question it was written to pose" + +# Blast radius: the mutation is confined to one of the two implementations. +expect "test_matches_a_route_with_a_parameter[regex]" FAILED \ + "the mutation stopped reaching the code the test covers" +expect "test_matches_a_route_with_a_parameter[segment]" PASSED \ + "the mutation leaked into the other implementation, so the fixture no longer shows a blast radius" + +# The legitimate survivor: asserting emptiness is satisfied by an empty implementation. +expect "test_empty_path_matches_nothing[regex]" PASSED \ + "the fixture no longer contains a test that correctly survives the mutation" + +if [ "$status" -ne 0 ]; then + echo + cat "$report" + echo "FAIL: the fixture is no longer the trap the eval needs." >&2 + exit 1 +fi + +echo "PASS: the fixture's contract test still cannot fail." diff --git a/skills/prove-the-test-fails/evals/check_wiring.py b/skills/prove-the-test-fails/evals/check_wiring.py new file mode 100644 index 0000000..86c6d2f --- /dev/null +++ b/skills/prove-the-test-fails/evals/check_wiring.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Checks the skill's frontmatter and its marketplace entry. + +The marketplace manifest is shared, so another skill's pull request can edit this +skill out of it without touching a file in this directory. Nothing here needs +credentials or a network. +""" + +import json +import re +import sys +from pathlib import Path + +SKILL_DIR = Path(__file__).resolve().parent.parent +SKILL_NAME = SKILL_DIR.name +REPO_ROOT = SKILL_DIR.parent.parent +MARKETPLACE = REPO_ROOT / ".claude-plugin" / "marketplace.json" + +failures: list[str] = [] + + +def require(condition: object, message: str) -> None: + if condition: + print(f" ok {message}") + else: + print(f" FAIL {message}") + failures.append(message) + + +frontmatter = re.match(r"---\n(.*?)\n---\n", (SKILL_DIR / "SKILL.md").read_text(), re.S) +require(frontmatter, "SKILL.md opens with YAML frontmatter") + +if frontmatter: + fields = dict(re.findall(r"^([a-z-]+):[ ]*(.+)$", frontmatter.group(1), re.M)) + require(fields.get("name") == SKILL_NAME, f"frontmatter name is {SKILL_NAME}") + require(len(fields.get("description", "")) > 80, "description is long enough to trigger on") + +entries = [ + entry + for entry in json.loads(MARKETPLACE.read_text())["plugins"] + if f"./skills/{SKILL_NAME}" in entry.get("skills", []) +] +require(len(entries) == 1, "exactly one marketplace entry loads this skill") + +if len(entries) == 1: + entry = entries[0] + require(entry["source"] == "./", "the entry's source is the marketplace root") + require( + entry.get("strict") is False, + "the entry is strict: false, since the root has no plugin.json", + ) + require( + "version" not in entry, "the entry carries no version, so it resolves from the commit SHA" + ) + require(entry.get("description"), "the entry has a description for the plugin picker") + +if failures: + sys.exit(f"FAIL: {len(failures)} wiring problem(s).") +print("PASS: the skill is packaged and listed correctly.") diff --git a/skills/prove-the-test-fails/evals/fixture/pyproject.toml b/skills/prove-the-test-fails/evals/fixture/pyproject.toml new file mode 100644 index 0000000..875366f --- /dev/null +++ b/skills/prove-the-test-fails/evals/fixture/pyproject.toml @@ -0,0 +1,3 @@ +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] diff --git a/skills/prove-the-test-fails/evals/fixture/src/routing.py b/skills/prove-the-test-fails/evals/fixture/src/routing.py new file mode 100644 index 0000000..d5c7f5c --- /dev/null +++ b/skills/prove-the-test-fails/evals/fixture/src/routing.py @@ -0,0 +1,64 @@ +"""Two implementations of one route matcher over one route table. + +Both are expected to satisfy the same result contract: for a given path they +return `Match` objects for the routes that accept it, most specific first. +""" + +import re +from dataclasses import dataclass + +ROUTES: list[tuple[int, str]] = [ + (1, "/users"), + (2, "/users/{id}"), + (3, "/users/{id}/settings"), + (4, "/health"), +] + + +@dataclass(frozen=True) +class Match: + """One route that accepts the path, and how specific that route is.""" + + route_id: int + pattern: str + specificity: int + + +def _is_parameter(segment: str) -> bool: + return segment.startswith("{") and segment.endswith("}") + + +def _specificity(pattern: str) -> int: + return sum(1 for segment in pattern.split("/") if segment and not _is_parameter(segment)) + + +def _ranked(matches: list[Match]) -> list[Match]: + return sorted(matches, key=lambda match: (-match.specificity, match.route_id)) + + +def regex_router(path: str) -> list[Match]: + """Compiled patterns: each route becomes a regex the whole path must match.""" + matches = [] + for route_id, pattern in ROUTES: + expression = "/".join( + "[^/]+" if _is_parameter(segment) else re.escape(segment) + for segment in pattern.split("/") + ) + if re.fullmatch(expression, path): + matches.append(Match(route_id, pattern, _specificity(pattern))) + return _ranked(matches) + + +def segment_router(path: str) -> list[Match]: + """Segment walk: the path is split on / and compared one segment at a time.""" + segments = path.split("/") + matches = [] + for route_id, pattern in ROUTES: + expected = pattern.split("/") + if len(expected) != len(segments): + continue + if all( + (_is_parameter(want) and got) or want == got for want, got in zip(expected, segments) + ): + matches.append(Match(route_id, pattern, _specificity(pattern))) + return _ranked(matches) diff --git a/skills/prove-the-test-fails/evals/fixture/tests/test_contract.py b/skills/prove-the-test-fails/evals/fixture/tests/test_contract.py new file mode 100644 index 0000000..563db16 --- /dev/null +++ b/skills/prove-the-test-fails/evals/fixture/tests/test_contract.py @@ -0,0 +1,25 @@ +"""The suite guarding the two route matchers.""" + +import pytest +from routing import regex_router, segment_router + +ROUTERS = {"regex": regex_router, "segment": segment_router} + + +@pytest.mark.parametrize("router", sorted(ROUTERS)) +def test_matches_a_route_with_a_parameter(router: str) -> None: + matches = ROUTERS[router]("/users/42") + assert [match.route_id for match in matches] == [2] + + +@pytest.mark.parametrize("router", sorted(ROUTERS)) +def test_empty_path_matches_nothing(router: str) -> None: + assert ROUTERS[router]("") == [] + + +def test_routers_satisfy_the_same_contract() -> None: + """Both routers return the same kind of result for the same path.""" + path = "/orders/7" + assert {type(match) for match in regex_router(path)} == { + type(match) for match in segment_router(path) + } diff --git a/skills/prove-the-test-fails/evals/prompt.md b/skills/prove-the-test-fails/evals/prompt.md new file mode 100644 index 0000000..f9348a6 --- /dev/null +++ b/skills/prove-the-test-fails/evals/prompt.md @@ -0,0 +1,14 @@ +This repository has two implementations of one route matcher, `regex_router` and +`segment_router`, and a suite in `tests/test_contract.py`. All five tests pass. + +Before code review leans on that suite, establish whether it actually guards the rule the +project cares about: that both implementations satisfy one shared result contract. + +You may change anything you like while you work, as long as you leave the repository +exactly as you found it. Then write your conclusion to `VERDICT.txt` in the repository +root. The first line must be exactly one of: + +- `GUARDED` if the suite would go red when an implementation stops satisfying the contract +- `UNGUARDED` if it would stay green + +Use the rest of the file for what you did and what you observed. diff --git a/skills/prove-the-test-fails/evals/run_eval.sh b/skills/prove-the-test-fails/evals/run_eval.sh new file mode 100755 index 0000000..8959078 --- /dev/null +++ b/skills/prove-the-test-fails/evals/run_eval.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# The agentic eval for the prove-the-test-fails skill. Costs LLM tokens. +# +# Copies the fixture into a scratch git repository, makes the skill available to a +# headless Claude Code run as a project skill, and asks the question the skill exists to +# answer: can this test fail? Grading is on behaviour, not prose: +# +# 1. the verdict is UNGUARDED, which is only knowable by breaking an implementation +# 2. the tracked files are byte-identical to the starting commit, so any mutation the +# agent applied was reverted +# 3. the suite is green again at the end +# +# --control runs the same task with the skill absent, which is how the eval shows the +# guidance changed anything. +# +# Usage: run_eval.sh [--control] [--model ] +set -euo pipefail + +EVALS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SKILL_DIR="$(dirname "$EVALS_DIR")" +SKILL_NAME="$(basename "$SKILL_DIR")" +CLAUDE_BIN="${CLAUDE_BIN:-claude}" +PYTEST=(uv run --no-project --with pytest pytest -q --tb=no -p no:cacheprovider) + +with_skill=1 +model="" +while [ $# -gt 0 ]; do + case "$1" in + --control) with_skill=0 ;; + --model) model="$2"; shift ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac + shift +done + +# The transcript lives beside the repository rather than in it, so the agent never sees +# its own output as a file in the tree it is auditing. +run_dir="$(mktemp -d)" +work="$run_dir/repo" +log="$run_dir/agent.log" +mkdir -p "$work" +cp -R "$EVALS_DIR/fixture/." "$work/" + +git -C "$work" init -q +git -C "$work" add -A +git -C "$work" -c user.name=eval -c user.email=eval@localhost \ + commit -qm "the fixture, green" +base="$(git -C "$work" rev-parse HEAD)" + +# The skill is on disk but outside git, so a dirty tree can only mean the agent left +# something behind. +if [ "$with_skill" -eq 1 ]; then + mkdir -p "$work/.claude/skills/$SKILL_NAME" + cp "$SKILL_DIR/SKILL.md" "$work/.claude/skills/$SKILL_NAME/SKILL.md" + echo ".claude/" >> "$work/.git/info/exclude" + echo "==> running with the skill available" +else + echo "==> control run: the skill is absent" +fi + +claude_args=(-p "$(cat "$EVALS_DIR/prompt.md")" + --permission-mode acceptEdits + --allowedTools "Bash Read Edit Write Grep Glob") +if [ -n "$model" ]; then + claude_args+=(--model "$model") +fi + +(cd "$work" && "$CLAUDE_BIN" "${claude_args[@]}") 2>&1 | tee "$log" || true + +echo +echo "==> grading" +status=0 +check() { + local what=$1 + shift + if "$@" > /dev/null 2>&1; then + echo " ok $what" + else + echo " FAIL $what" + status=1 + fi +} + +verdict="$(head -n 1 "$work/VERDICT.txt" 2>/dev/null | tr -d '[:space:]' || true)" +check "verdict is UNGUARDED (got: '${verdict:-}')" \ + test "$verdict" = UNGUARDED +check "tracked files are unchanged, so any mutation was reverted" \ + git -C "$work" diff --quiet "$base" -- src tests pyproject.toml +suite_green() { (cd "$work" && "${PYTEST[@]}"); } +check "the suite is green again" suite_green + +if [ "$status" -ne 0 ]; then + echo "FAIL: transcript and repository left at $run_dir" >&2 + exit 1 +fi + +rm -rf "$run_dir" +echo "PASS"