diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 4c5f5d6..648be7f 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -15,6 +15,13 @@ "name": "tastytrade", "source": "./plugins/tastytrade", "description": "Inspect TastyTrade brokerage accounts, positions, market data, option chains, and transactions from Claude Code. Order placement is gated off by default." + }, + { + "name": "parallel-agent-isolation", + "source": "./", + "strict": false, + "skills": ["./skills/parallel-agent-isolation"], + "description": "Decide what stateful resources concurrent agents will share, and pick isolation, serialisation, or serial re-verification before dispatching them." } ] } diff --git a/.github/workflows/skill-parallel-agent-isolation.yml b/.github/workflows/skill-parallel-agent-isolation.yml new file mode 100644 index 0000000..df68a96 --- /dev/null +++ b/.github/workflows/skill-parallel-agent-isolation.yml @@ -0,0 +1,53 @@ +name: skill parallel-agent-isolation + +# One workflow per skill, matching the per-component paths filters used for the +# MCP servers, so a change to one skill never runs another's checks. +# +# marketplace.json is in the filter because it is the shared file: an entry +# edited by another skill's pull request can silently stop this one loading. +on: + pull_request: + paths: + - "skills/parallel-agent-isolation/**" + - ".claude-plugin/marketplace.json" + - ".github/workflows/skill-parallel-agent-isolation.yml" + push: + branches: [main] + paths: + - "skills/parallel-agent-isolation/**" + - ".claude-plugin/marketplace.json" + - ".github/workflows/skill-parallel-agent-isolation.yml" + +defaults: + run: + working-directory: skills/parallel-agent-isolation + +jobs: + wiring: + # Packaging, marketplace entry, and case set. No credentials, no tokens. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: make check + - run: npm install -g @anthropic-ai/claude-code + - name: Validate the marketplace manifest + working-directory: . + run: claude plugin validate . --strict + + evals: + # Behavioural: does the skill load on its own, and does the answer resolve + # the shared resource. Costs Anthropic tokens, so it skips without a key + # (fork PRs get no secrets). + runs-on: ubuntu-latest + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + steps: + - uses: actions/checkout@v4 + - name: Run the behavioural evals + run: | + if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + echo "ANTHROPIC_API_KEY absent (fork PR?); skipping the behavioural evals." + exit 0 + fi + npm install -g @anthropic-ai/claude-code + make evals diff --git a/README.md b/README.md index e03955b..d5567d6 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,18 @@ Formerly `walkerhughes/mcps`, back when it only held MCP servers. They follow Honeycomb's [MCP, easy as 1-2-3](https://www.honeycomb.io/blog/mcp-easy-as-1-2-3) guidance: a few curated tools built around real questions rather than raw API endpoints, responses shaped for a model instead of a UI, and typed schemas that steer the model toward valid calls. +## Skills + +Skills live in top-level `skills/`, one directory each, and ship through the marketplace as their own installable entries. + +| Skill | What it decides | +|-------|-----------------| +| [`parallel-agent-isolation`](skills/parallel-agent-isolation/) | What stateful resources concurrent agents will share, and whether to isolate, serialise, or re-verify serially, before dispatching them. | + +A skill's marketplace entry sets `source: "./"` with a `skills` path pointing at its own directory, so several skills can share the one top-level folder without loading each other, and `strict: false` because the repository root has no `plugin.json` to be the authority. Entries deliberately carry no `version`: Claude Code then resolves the version from the commit SHA, so every change reaches installed copies without a manual bump, and the stale-cache trap described below does not apply. + +Each skill directory carries its own checks, run from that directory: `make check` for packaging and wiring, which needs no credentials, and `make evals` for behaviour, which costs tokens. CI runs both per skill through `paths` filters, as it does for the servers. + ## Install a plugin Run these as two separate commands, not as one paste: the first opens a prompt that expects only the `owner/repo`. @@ -44,9 +56,11 @@ Plugins require [`uv`](https://docs.astral.sh/uv/) on your PATH. The first launc ``` claude/ ├── .claude-plugin/ # marketplace manifest -└── plugins/ - ├── harbor-hub/ - └── tastytrade/ +├── plugins/ +│ ├── harbor-hub/ +│ └── tastytrade/ +└── skills/ + └── parallel-agent-isolation/ ``` Plugins live under `plugins/`, one directory each, named for the platform they talk to rather than for being an MCP server. Skills and other components get their own top-level directories as they arrive. diff --git a/skills/parallel-agent-isolation/Makefile b/skills/parallel-agent-isolation/Makefile new file mode 100644 index 0000000..8a78f43 --- /dev/null +++ b/skills/parallel-agent-isolation/Makefile @@ -0,0 +1,9 @@ +.PHONY: check evals + +# Packaging and wiring. No API key, no cost. +check: + python3 evals/check_wiring.py + +# Behavioural eval. Needs Claude Code on PATH and working credentials. +evals: + python3 evals/run_evals.py diff --git a/skills/parallel-agent-isolation/SKILL.md b/skills/parallel-agent-isolation/SKILL.md new file mode 100644 index 0000000..404e804 --- /dev/null +++ b/skills/parallel-agent-isolation/SKILL.md @@ -0,0 +1,70 @@ +--- +name: parallel-agent-isolation +description: Decide what stateful resources concurrent agents will share, then isolate, serialise, or re-verify serially before dispatching them. Worktrees isolate files, not services, so parallel agents collide on one database, one simulator, or one state file and both still report green. +when_to_use: Use when two or more agents will run concurrently AND their work touches something stateful outside the filesystem, such as a database, a service stack, an integration or end-to-end suite, a search index, a message broker, a cache, a fixed port, a device or simulator, a shared cloud environment or IaC state file, or a shared account or API key. Also use when judging a finished parallel agent's report, especially one saying it could not get its own instance of such a resource and used the one already running, in any wording, including "port is already allocated", "address already in use", "container name is already in use", or "no free simulator". Do not use when concurrent agents only read and write files, such as editing docs or independent source modules, since worktrees already isolate that. +--- + +# Parallel agents share more than the filesystem + +Decide what concurrently running agents will contend for before dispatching +them, and treat any result produced under contention as unverified. + +A git worktree isolates files and nothing else. Agents in separate worktrees +still query the one database, boot the one device, hold the one lock, and spend +the one account's quota. Shared state is whatever exists as a single instance +and carries changes between calls, so what one agent does lands where another +will read it. + +## The failure is silent + +Contention does not raise an error. Two suites against one database interleave, +and a fixture truncating tables between tests deletes the other run's rows +mid-test, so both go green. Two runs driving one device tap each other's +screens. Two applies against one state file each plan against a world the other +has already changed. All of them report success, indistinguishably from an +honest run. + +So this is decided before dispatch, not diagnosed after. Afterwards there is +nothing to find, only the question of whether the result was produced under +contention, and a result produced under contention is unverified whatever it +says. + +## The pre-dispatch workflow + +**1. Name what will be contended for.** Ask what stateful things outside the +filesystem these agents will share, and name each one separately. The obvious +resource usually hides a second, and resolving one leaves the other shared. If +the honest answer is nothing, the work is file-local: dispatch in parallel and +stop here. + +**2. Choose how the contention resolves.** Any of the four chosen deliberately +beats meeting the collision later. + +- **Isolate.** Each agent gets its own instance, ports, schema, or account. Fits + when duplicating the resource is cheap next to the work it unblocks. +- **Serialise the stateful step.** Agents run in parallel, and the dispatcher + lets one at a time through the step that touches the resource. Fits when that + step is short next to the work around it. A built-in lock is not this: it + covers only the thing it guards, leaving whatever that thing mutates still + shared, so the serialised region has to span the change and the verification + that depends on it. +- **Parallel, then verify serially.** Take every agent's result as advisory and + re-run the verification yourself, one at a time. Fits when verification is + cheap, and it is the only option that still holds when an agent ignores its + instructions, so prefer it when unsure. +- **Split the work.** One agent owns everything touching the resource; the rest + parallelise around it. Fits when only some tasks need the resource at all. + +**3. Dispatch with the reporting requirement.** Require each agent to report how +it verified its work, not just the outcome: the command it ran, and whether +every service, device, environment, and account that command touched was its +alone. Say up front that a result which cannot answer that is advisory. A green +result that does not say how it was produced is unverified. + +## The anti-pattern + +An agent reports that it could not obtain its own instance of a shared resource, +used the one already running, and finished green. That is a corrupted run +presented as a workaround. It invalidates its own result rather than excusing +it, and casts the same doubt over every other agent that was using the resource +at the time. diff --git a/skills/parallel-agent-isolation/evals/README.md b/skills/parallel-agent-isolation/evals/README.md new file mode 100644 index 0000000..ed0db62 --- /dev/null +++ b/skills/parallel-agent-isolation/evals/README.md @@ -0,0 +1,61 @@ +# parallel-agent-isolation evals + +Two checks, in cost order. + +```bash +make check # packaging and wiring, no API key, no cost +make evals # behavioural, needs Claude Code on PATH and credentials +``` + +## `make check` + +`check_wiring.py` reads the skill's frontmatter, its marketplace entry, and the +case set. It exists because every way a skill breaks in packaging is silent: an +unclosed frontmatter fence, a `name` that no longer matches the directory, a +marketplace entry whose `skills` path points somewhere else, a `version` pinned +in the entry so installed copies never refresh. In each case the skill installs +and simply never loads. Stdlib only, so it runs anywhere. + +## `make evals` + +`run_evals.py` runs each case as headless Claude Code in a throwaway workspace +holding this skill and nothing else, with `--setting-sources project` so a +personal skill on the developer's machine cannot stand in for the one under +test. Every case asserts on skill invocation; the triggering cases additionally +grade the answer against a rubric using a second, tool-less Claude Code call +that returns a structured verdict and never sees the skill. + +The triggering cases deliberately sit in different ecosystems and share different +kinds of state, because a skill that recognised only containers and databases +would pass a case set drawn from one stack while being useless on the next one. + +| Case | Stack, and what is shared | Skill must | The answer must | +| --- | --- | --- | --- | +| `shared-database-dispatch` | Go, one hosted staging database | load | resolve the shared instance by isolating, serialising, re-verifying serially, or giving the stateful work to one agent, rather than dispatching both agents at the suite concurrently | +| `shared-infra-state` | Terraform, one remote state file and one sandbox account | load | resolve the shared state and account the same way, rather than letting three applies race through the lock | +| `green-report-collision` | iOS, one booted simulator | load | treat a green suite reported alongside "a second simulator would not boot" as invalid, extend the doubt to the other agent's green result, and re-run serially | +| `file-only-parallel` | any, nothing shared | stay out | (not graded) | + +`file-only-parallel` is the honesty check, and it earns its place. A description +that fires on every mention of parallel agents would pass the triggering cases +while making the skill noise, so one case dispatches three agents over +documentation edits and requires the skill to stay out of it. The first draft of +the description failed exactly there, and so did a later attempt to shorten the +clause that excludes file-only work. + +Whether a description triggers is sampled behaviour, so each case runs three +times and every run must hold. Cost is a dollar or so a full run on `sonnet`, +capped per call by `--max-budget-usd`. `EVAL_RUNS`, `EVAL_MODEL`, +`EVAL_MAX_BUDGET_USD`, and `EVAL_TIMEOUT_SEC` override the defaults. Pass case +names as arguments to run a subset. + +## Why not the Harbor harness + +The [harbor-hub evals](../../../plugins/harbor-hub/evals/) are Harbor tasks: a +container image, an MCP server declared in `task.toml`, and a verifier that +recomputes ground truth from the hub. That shape fits a capability surface with +a live backend to compare against. This skill has no server, no backend, and +nothing to recompute. What it has is a claim about what a model does at dispatch +time, so the eval runs the model and looks. Wrapping that in Harbor would add an +image build, a Modal or Docker dependency, and a Harbor API key to a check whose +whole content is one prompt and one grading rubric. diff --git a/skills/parallel-agent-isolation/evals/cases/file-only-parallel/case.json b/skills/parallel-agent-isolation/evals/cases/file-only-parallel/case.json new file mode 100644 index 0000000..f68724d --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/file-only-parallel/case.json @@ -0,0 +1,4 @@ +{ + "expect_skill": false, + "rubric": null +} diff --git a/skills/parallel-agent-isolation/evals/cases/file-only-parallel/prompt.md b/skills/parallel-agent-isolation/evals/cases/file-only-parallel/prompt.md new file mode 100644 index 0000000..cd2223a --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/file-only-parallel/prompt.md @@ -0,0 +1,7 @@ +I want to dispatch three subagents in parallel over this repo, each in its own +git worktree. One writes the module docstrings for `src/parsing/`, one does the +same for `src/rendering/`, and one rewrites the README's install section. + +None of them run tests or start anything. Write me the dispatch plan. + +Do not run anything. Just give me the plan. diff --git a/skills/parallel-agent-isolation/evals/cases/green-report-collision/case.json b/skills/parallel-agent-isolation/evals/cases/green-report-collision/case.json new file mode 100644 index 0000000..778e8b5 --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/green-report-collision/case.json @@ -0,0 +1,8 @@ +{ + "expect_skill": true, + "rubric": [ + "The answer must say the work is not done, and must treat the reuse of the already-booted simulator as invalidating the agent's green test result rather than as a harmless note or a minor warning.", + "The answer must state or clearly imply that both agents ran against the same simulator, so both green results are untrustworthy, not only the one that reported the problem.", + "The answer must call for re-running the suites serially against a known-clean simulator before trusting either result." + ] +} diff --git a/skills/parallel-agent-isolation/evals/cases/green-report-collision/prompt.md b/skills/parallel-agent-isolation/evals/cases/green-report-collision/prompt.md new file mode 100644 index 0000000..0d90cdf --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/green-report-collision/prompt.md @@ -0,0 +1,11 @@ +Two subagents I dispatched in parallel have finished. They worked in separate +git worktrees on the same iOS app. Here is what the second one sent back: + +> Fixed the checkout crash. Ran the UI suite with `xcodebuild test`: 84 passed, +> 0 failed. Note: a second simulator would not boot, so I ran against the +> iPhone 16 simulator that was already booted and installed over the build that +> was on it. Suite is green. + +The first agent reported green as well. Am I done? Tell me what to do next. + +Do not run anything. Just answer. diff --git a/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/case.json b/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/case.json new file mode 100644 index 0000000..2ec97ee --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/case.json @@ -0,0 +1,8 @@ +{ + "expect_skill": true, + "rubric": [ + "The plan must recognise that the two agents would share the one staging Postgres instance even though each has its own git worktree, and must not dispatch both agents to run the e2e suite concurrently against it.", + "The plan must resolve that sharing in at least one of these ways: give each agent its own database instance, schema, or namespace; serialise the e2e step so only one agent runs it at a time; keep the agents parallel but re-run the suites serially afterwards and treat the agents' own results as unverified; or give the e2e work to a single agent.", + "It is not enough to merely mention worktrees, or to say the agents should 'be careful' or 'coordinate' without naming one of those resolutions." + ] +} diff --git a/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/prompt.md b/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/prompt.md new file mode 100644 index 0000000..cadeb0c --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/prompt.md @@ -0,0 +1,14 @@ +I maintain a Go service. Its end-to-end tests run with `go test -tags e2e ./...` +against our shared staging Postgres, a managed instance in our cloud account +that everyone on the team points at. There is no local copy of it. The unit +tests need nothing. + +I have two independent pieces of work queued: adding pagination to the orders +endpoint, and fixing a timezone bug in the reporting query. Each one needs the +e2e suite to pass before I will look at it. + +I want to hand both to subagents at the same time, each in its own git worktree, +and have them report back. Write me the dispatch plan: how many agents, what +each one does, and what I put in their prompts. + +Do not run anything. Just give me the plan. diff --git a/skills/parallel-agent-isolation/evals/cases/shared-infra-state/case.json b/skills/parallel-agent-isolation/evals/cases/shared-infra-state/case.json new file mode 100644 index 0000000..f3a62c2 --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/shared-infra-state/case.json @@ -0,0 +1,8 @@ +{ + "expect_skill": true, + "rubric": [ + "The plan must recognise that the three agents would share the one Terraform state file and the one sandbox account even though each has its own git worktree, and must not dispatch all three to apply concurrently against that single state and account.", + "The plan must resolve that sharing in at least one of these ways: give each agent its own state, workspace, or sandbox account so the applies do not meet; serialise the apply and smoke-test step so only one agent runs it at a time; keep the agents parallel on the code but do the applies and verification serially afterwards and treat the agents' own results as unverified; or give all the applying to a single agent.", + "It is not enough to merely mention worktrees, or to rely on the state lock alone, or to say the agents should 'be careful' or 'coordinate' without naming one of those resolutions." + ] +} diff --git a/skills/parallel-agent-isolation/evals/cases/shared-infra-state/prompt.md b/skills/parallel-agent-isolation/evals/cases/shared-infra-state/prompt.md new file mode 100644 index 0000000..1681b4e --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/shared-infra-state/prompt.md @@ -0,0 +1,14 @@ +Our infrastructure is a Terraform repo. The backend is one S3 bucket with a +DynamoDB lock table, and everything applies into a single sandbox cloud account +we use to try changes before production. Verifying a change means applying it in +the sandbox and then running a smoke script against the deployed endpoint. + +Three unrelated changes are queued: a new CDN cache policy, a database instance +resize, and a tightened security group. I want three subagents in parallel, each +in its own git worktree, each applying and smoke-testing its own change and +reporting back. + +Write me the dispatch plan: how many agents, what each one does, and what I put +in their prompts. + +Do not run anything. Just give me the plan. diff --git a/skills/parallel-agent-isolation/evals/check_wiring.py b/skills/parallel-agent-isolation/evals/check_wiring.py new file mode 100644 index 0000000..706d790 --- /dev/null +++ b/skills/parallel-agent-isolation/evals/check_wiring.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Structural checks for the parallel-agent-isolation skill. + +A skill fails silently when its packaging is wrong: an unparsed frontmatter +block, a name that no longer matches the directory, or a marketplace entry that +does not point at the skill directory all leave the skill installed and never +loaded. These checks run without an API key so they can gate every PR. + +Stdlib only, and the frontmatter is parsed by hand, because a top-level skill +directory has no dependency manifest to hang PyYAML off. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +SKILL_DIR = Path(__file__).resolve().parent.parent +REPO_ROOT = SKILL_DIR.parent.parent +SKILL_NAME = SKILL_DIR.name + +# Claude Code truncates the combined description and when_to_use text at this +# many characters in the skill listing, so anything past it never reaches the +# model that decides whether to load the skill. +LISTING_CAP = 1536 + +failures: list[str] = [] + + +def check(condition: bool, message: str) -> bool: + if not condition: + failures.append(message) + return condition + + +def parse_frontmatter(text: str) -> dict[str, str]: + """Read a `key: value` frontmatter block, one entry per line.""" + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + failures.append("SKILL.md does not open with a --- frontmatter fence") + return {} + try: + end = lines.index("---", 1) + except ValueError: + failures.append("SKILL.md frontmatter is never closed with ---") + return {} + + fields: dict[str, str] = {} + for lineno, line in enumerate(lines[1:end], start=2): + if not line.strip(): + continue + key, sep, value = line.partition(":") + if not sep or not key.strip() or key.startswith((" ", "\t")): + failures.append( + f"SKILL.md line {lineno} is not a single-line `key: value` entry: {line!r}" + ) + continue + fields[key.strip()] = value.strip() + return fields + + +def check_skill_md() -> None: + path = SKILL_DIR / "SKILL.md" + if not check(path.is_file(), f"missing {path}"): + return + fields = parse_frontmatter(path.read_text()) + + check( + fields.get("name") == SKILL_NAME, + f"frontmatter name is {fields.get('name')!r}, expected the directory name {SKILL_NAME!r}", + ) + check( + bool(fields.get("description")), + "frontmatter has no description, so nothing tells Claude when to load the skill", + ) + + listing = f"{fields.get('description', '')} {fields.get('when_to_use', '')}".strip() + check( + len(listing) <= LISTING_CAP, + f"description plus when_to_use is {len(listing)} characters and is truncated at {LISTING_CAP}", + ) + + body = path.read_text().split("---", 2)[-1] + check( + bool(body.strip()), + "SKILL.md has frontmatter but no body, so the skill loads and says nothing", + ) + + +def check_marketplace_entry() -> None: + path = REPO_ROOT / ".claude-plugin" / "marketplace.json" + if not check(path.is_file(), f"missing {path}"): + return + manifest = json.loads(path.read_text()) + + entries = [p for p in manifest.get("plugins", []) if p.get("name") == SKILL_NAME] + if not check( + len(entries) == 1, + f"expected exactly one marketplace entry named {SKILL_NAME!r}, found {len(entries)}", + ): + return + entry = entries[0] + + check( + entry.get("source") == "./", + f"entry source is {entry.get('source')!r}, expected './' for a marketplace-root skill", + ) + check( + entry.get("strict") is False, + "entry must set strict to false: there is no plugin.json at the repository root to be the authority", + ) + check( + entry.get("skills") == [f"./skills/{SKILL_NAME}"], + f"entry skills is {entry.get('skills')!r}, expected ['./skills/{SKILL_NAME}']", + ) + check( + bool(entry.get("description")), + "entry has no description, so the plugin listing shows nothing", + ) + check( + "version" not in entry, + "entry pins a version; omit it so the commit SHA is the version and installed copies refresh on every change", + ) + + +def check_cases() -> None: + cases_dir = SKILL_DIR / "evals" / "cases" + cases = sorted(p for p in cases_dir.iterdir() if p.is_dir()) if cases_dir.is_dir() else [] + if not check(len(cases) >= 2, f"expected at least two eval cases in {cases_dir}"): + return + + expectations = set() + for case in cases: + if not check((case / "prompt.md").is_file(), f"{case.name}: missing prompt.md"): + continue + spec_path = case / "case.json" + if not check(spec_path.is_file(), f"{case.name}: missing case.json"): + continue + spec = json.loads(spec_path.read_text()) + if check( + isinstance(spec.get("expect_skill"), bool), + f"{case.name}: expect_skill must be a boolean", + ): + expectations.add(spec["expect_skill"]) + if spec["expect_skill"]: + check( + bool(spec.get("rubric")), + f"{case.name}: a triggering case needs a rubric to grade the answer against", + ) + + check( + expectations == {True, False}, + "the case set needs both a case the skill must load for and one it must stay out of; a description that " + "triggers on everything is not a working description", + ) + + +check_skill_md() +check_marketplace_entry() +check_cases() + +for failure in failures: + print(f"FAIL: {failure}", file=sys.stderr) +if failures: + sys.exit(1) +print(f"ok: {SKILL_NAME} packaging, marketplace entry, and eval cases") diff --git a/skills/parallel-agent-isolation/evals/run_evals.py b/skills/parallel-agent-isolation/evals/run_evals.py new file mode 100644 index 0000000..25d4d1e --- /dev/null +++ b/skills/parallel-agent-isolation/evals/run_evals.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Behavioural eval for the parallel-agent-isolation skill. + +Each case runs headless Claude Code in a throwaway workspace that contains only +this skill, and checks two things: whether the skill loaded on its own, and +whether the answer resolves the shared-resource problem instead of dispatching +naively. A rubric is graded by a second, tool-less Claude Code call returning a +structured verdict. + +Both halves matter. A skill that never loads is inert whatever its body says, +and a skill that loads for every parallel dispatch is noise, so one case must +not trigger it at all. + +Usage: python3 run_evals.py [case-name ...] +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +SKILL_DIR = Path(__file__).resolve().parent.parent +SKILL_NAME = SKILL_DIR.name +CASES_DIR = Path(__file__).resolve().parent / "cases" + +MODEL = os.environ.get("EVAL_MODEL", "sonnet") +BUDGET_USD = os.environ.get("EVAL_MAX_BUDGET_USD", "1.00") +TIMEOUT_SEC = int(os.environ.get("EVAL_TIMEOUT_SEC", "600")) +# Whether a description triggers is a sampled behaviour, so one run of a case +# says little. Every run of a case must hold for the case to pass. +RUNS = int(os.environ.get("EVAL_RUNS", "3")) + +VERDICT_SCHEMA = { + "type": "object", + "properties": { + "verdict": {"type": "string", "enum": ["pass", "fail"]}, + "reason": {"type": "string"}, + }, + "required": ["verdict", "reason"], +} + + +class EvalError(RuntimeError): + pass + + +def run_claude(prompt: str, cwd: Path, extra: list[str]) -> list[dict]: + """Run headless Claude Code and return the parsed stream-json events.""" + command = [ + "claude", + "-p", + prompt, + "--output-format", + "stream-json", + "--verbose", + "--model", + MODEL, + # Only the project's own .claude/ is loaded, so a personal skill on the + # developer's machine cannot stand in for the one under test. + "--setting-sources", + "project", + "--strict-mcp-config", + "--no-session-persistence", + "--max-budget-usd", + BUDGET_USD, + *extra, + ] + result = subprocess.run(command, cwd=cwd, capture_output=True, text=True, timeout=TIMEOUT_SEC) + if result.returncode != 0: + raise EvalError(f"claude exited {result.returncode}: {result.stderr.strip()[:500]}") + + events = [json.loads(line) for line in result.stdout.splitlines() if line.strip()] + if not events: + raise EvalError("claude produced no output") + return events + + +def final_text(events: list[dict]) -> str: + for event in reversed(events): + if event.get("type") == "result": + if event.get("is_error"): + raise EvalError( + f"claude reported an error result: {str(event.get('result'))[:300]}" + ) + return event.get("result") or "" + raise EvalError("no result event in the stream") + + +def total_cost(events: list[dict]) -> float: + for event in reversed(events): + if event.get("type") == "result": + return float(event.get("total_cost_usd") or 0.0) + return 0.0 + + +def skill_was_loaded(events: list[dict]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + for block in event.get("message", {}).get("content", []): + if block.get("type") == "tool_use" and block.get("name") == "Skill": + if (block.get("input") or {}).get("skill") == SKILL_NAME: + return True + return False + + +def judge(rubric: list[str], answer: str, workspace: Path) -> tuple[bool, str]: + """Grade an answer against a rubric with a tool-less Claude Code call.""" + criteria = "\n".join(f"{i}. {line}" for i, line in enumerate(rubric, start=1)) + prompt = ( + "You are grading one answer against fixed criteria. Judge only what the answer says.\n" + "Return verdict 'pass' only if every criterion is met, otherwise 'fail'.\n" + "Keep reason to one sentence, and quote the answer where it decides the verdict.\n\n" + f"CRITERIA\n{criteria}\n\nANSWER\n{answer}\n" + ) + events = run_claude( + prompt, + workspace, + ["--tools", "", "--json-schema", json.dumps(VERDICT_SCHEMA)], + ) + try: + parsed = json.loads(final_text(events)) + except json.JSONDecodeError as exc: + raise EvalError(f"judge did not return the verdict schema: {exc}") from exc + # The judge occasionally trails a stray closing tag after its reason. + reason = re.sub(r"(\s*)+\s*$", "", " ".join(parsed["reason"].split())) + return parsed["verdict"] == "pass", reason + + +def run_case(case: Path, workspace: Path, judge_workspace: Path) -> tuple[bool, str, float]: + """Run one case RUNS times. Every run must hold, so one lucky pass is not a pass.""" + spec = json.loads((case / "case.json").read_text()) + prompt = (case / "prompt.md").read_text() + cost = 0.0 + + for attempt in range(1, RUNS + 1): + events = run_claude(prompt, workspace, ["--tools", "Skill"]) + cost += total_cost(events) + + loaded = skill_was_loaded(events) + if loaded != spec["expect_skill"]: + want = "load" if spec["expect_skill"] else "stay out of" + return False, f"run {attempt}/{RUNS}: the skill did not {want} this scenario", cost + + if not spec.get("rubric"): + continue + + passed, reason = judge(spec["rubric"], final_text(events), judge_workspace) + if not passed: + return False, f"run {attempt}/{RUNS}: {reason}", cost + + return True, f"{RUNS}/{RUNS} runs held", cost + + +def main() -> int: + if not shutil.which("claude"): + print("claude is not on PATH; install Claude Code to run these evals.", file=sys.stderr) + return 1 + + wanted = set(sys.argv[1:]) + cases = sorted( + p for p in CASES_DIR.iterdir() if p.is_dir() and (not wanted or p.name in wanted) + ) + if not cases: + print(f"no cases matched {sorted(wanted)}", file=sys.stderr) + return 1 + + with tempfile.TemporaryDirectory(prefix=f"{SKILL_NAME}-evals-") as tmp: + root = Path(tmp) + # The agent workspace holds this skill and nothing else, so a load is + # attributable to this skill's description rather than to the repository. + workspace = root / "workspace" + shutil.copytree( + SKILL_DIR, + workspace / ".claude" / "skills" / SKILL_NAME, + ignore=shutil.ignore_patterns("evals"), + ) + # The judge gets an empty workspace: it must not see the skill it grades against. + judge_workspace = root / "judge" + judge_workspace.mkdir() + + failed: list[str] = [] + spent = 0.0 + for case in cases: + try: + passed, reason, cost = run_case(case, workspace, judge_workspace) + except (EvalError, subprocess.TimeoutExpired) as exc: + passed, reason, cost = False, str(exc), 0.0 + spent += cost + print(f"{'PASS' if passed else 'FAIL'} {case.name}: {reason}", flush=True) + if not passed: + failed.append(case.name) + + print(f"\n{len(cases) - len(failed)}/{len(cases)} cases passed, ${spent:.2f} spent") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main())