From a5f9c5a61ad0b4b1fd0eb84f68864a0903b5f8c6 Mon Sep 17 00:00:00 2001 From: Walker Hughes <74113220+walkerhughes@users.noreply.github.com> Date: Sun, 2 Aug 2026 17:52:14 -0700 Subject: [PATCH 1/3] feat(skills): add the parallel-agent-isolation skill Encodes a failure that hit twice in one session: two agents dispatched concurrently, each in its own git worktree, both fell back to the one Postgres and OpenSearch already bound to their ports and ran their integration suites against a shared database. Fixtures that truncate tables between tests wiped each other's rows mid-run. Both agents reported green and neither run proved anything. The skill asks the one question that prevents it, before dispatch: what stateful thing outside the filesystem will these agents share. It offers four resolutions with the conditions each fits, names the collision signals that invalidate an agent's verification after the fact, and covers the containers, worktrees, and worktree-pinned branches left behind. Packaged as a top-level skills/ directory with its own marketplace entry using source "./" and a skills path, per the marketplace docs' pattern for several entries sharing one skills folder. The entry carries no version so Claude Code resolves it from the commit SHA and installed copies refresh without a manual bump. Two checks, both per-skill through paths filters. make check validates frontmatter, the marketplace entry, and the case set without credentials. make evals runs each case as headless Claude Code in a workspace holding only this skill, three times, and requires every run to hold: the two scenario cases must load the skill and resolve the shared resource, and a file-only dispatch must not load it at all. That last case caught an over-broad description twice. --- .claude-plugin/marketplace.json | 7 + .../skill-parallel-agent-isolation.yml | 53 +++++ README.md | 20 +- skills/parallel-agent-isolation/Makefile | 9 + skills/parallel-agent-isolation/SKILL.md | 69 ++++++ .../parallel-agent-isolation/evals/README.md | 56 +++++ .../evals/cases/file-only-parallel/case.json | 4 + .../evals/cases/file-only-parallel/prompt.md | 7 + .../cases/green-report-collision/case.json | 8 + .../cases/green-report-collision/prompt.md | 11 + .../cases/shared-database-dispatch/case.json | 8 + .../cases/shared-database-dispatch/prompt.md | 13 ++ .../evals/check_wiring.py | 167 ++++++++++++++ .../evals/run_evals.py | 205 ++++++++++++++++++ 14 files changed, 634 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/skill-parallel-agent-isolation.yml create mode 100644 skills/parallel-agent-isolation/Makefile create mode 100644 skills/parallel-agent-isolation/SKILL.md create mode 100644 skills/parallel-agent-isolation/evals/README.md create mode 100644 skills/parallel-agent-isolation/evals/cases/file-only-parallel/case.json create mode 100644 skills/parallel-agent-isolation/evals/cases/file-only-parallel/prompt.md create mode 100644 skills/parallel-agent-isolation/evals/cases/green-report-collision/case.json create mode 100644 skills/parallel-agent-isolation/evals/cases/green-report-collision/prompt.md create mode 100644 skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/case.json create mode 100644 skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/prompt.md create mode 100644 skills/parallel-agent-isolation/evals/check_wiring.py create mode 100644 skills/parallel-agent-isolation/evals/run_evals.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 4c5f5d6..648be7f 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -15,6 +15,13 @@ "name": "tastytrade", "source": "./plugins/tastytrade", "description": "Inspect TastyTrade brokerage accounts, positions, market data, option chains, and transactions from Claude Code. Order placement is gated off by default." + }, + { + "name": "parallel-agent-isolation", + "source": "./", + "strict": false, + "skills": ["./skills/parallel-agent-isolation"], + "description": "Decide what stateful resources concurrent agents will share, and pick isolation, serialisation, or serial re-verification before dispatching them." } ] } diff --git a/.github/workflows/skill-parallel-agent-isolation.yml b/.github/workflows/skill-parallel-agent-isolation.yml new file mode 100644 index 0000000..df68a96 --- /dev/null +++ b/.github/workflows/skill-parallel-agent-isolation.yml @@ -0,0 +1,53 @@ +name: skill parallel-agent-isolation + +# One workflow per skill, matching the per-component paths filters used for the +# MCP servers, so a change to one skill never runs another's checks. +# +# marketplace.json is in the filter because it is the shared file: an entry +# edited by another skill's pull request can silently stop this one loading. +on: + pull_request: + paths: + - "skills/parallel-agent-isolation/**" + - ".claude-plugin/marketplace.json" + - ".github/workflows/skill-parallel-agent-isolation.yml" + push: + branches: [main] + paths: + - "skills/parallel-agent-isolation/**" + - ".claude-plugin/marketplace.json" + - ".github/workflows/skill-parallel-agent-isolation.yml" + +defaults: + run: + working-directory: skills/parallel-agent-isolation + +jobs: + wiring: + # Packaging, marketplace entry, and case set. No credentials, no tokens. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: make check + - run: npm install -g @anthropic-ai/claude-code + - name: Validate the marketplace manifest + working-directory: . + run: claude plugin validate . --strict + + evals: + # Behavioural: does the skill load on its own, and does the answer resolve + # the shared resource. Costs Anthropic tokens, so it skips without a key + # (fork PRs get no secrets). + runs-on: ubuntu-latest + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + steps: + - uses: actions/checkout@v4 + - name: Run the behavioural evals + run: | + if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + echo "ANTHROPIC_API_KEY absent (fork PR?); skipping the behavioural evals." + exit 0 + fi + npm install -g @anthropic-ai/claude-code + make evals diff --git a/README.md b/README.md index e03955b..d5567d6 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,18 @@ Formerly `walkerhughes/mcps`, back when it only held MCP servers. They follow Honeycomb's [MCP, easy as 1-2-3](https://www.honeycomb.io/blog/mcp-easy-as-1-2-3) guidance: a few curated tools built around real questions rather than raw API endpoints, responses shaped for a model instead of a UI, and typed schemas that steer the model toward valid calls. +## Skills + +Skills live in top-level `skills/`, one directory each, and ship through the marketplace as their own installable entries. + +| Skill | What it decides | +|-------|-----------------| +| [`parallel-agent-isolation`](skills/parallel-agent-isolation/) | What stateful resources concurrent agents will share, and whether to isolate, serialise, or re-verify serially, before dispatching them. | + +A skill's marketplace entry sets `source: "./"` with a `skills` path pointing at its own directory, so several skills can share the one top-level folder without loading each other, and `strict: false` because the repository root has no `plugin.json` to be the authority. Entries deliberately carry no `version`: Claude Code then resolves the version from the commit SHA, so every change reaches installed copies without a manual bump, and the stale-cache trap described below does not apply. + +Each skill directory carries its own checks, run from that directory: `make check` for packaging and wiring, which needs no credentials, and `make evals` for behaviour, which costs tokens. CI runs both per skill through `paths` filters, as it does for the servers. + ## Install a plugin Run these as two separate commands, not as one paste: the first opens a prompt that expects only the `owner/repo`. @@ -44,9 +56,11 @@ Plugins require [`uv`](https://docs.astral.sh/uv/) on your PATH. The first launc ``` claude/ ├── .claude-plugin/ # marketplace manifest -└── plugins/ - ├── harbor-hub/ - └── tastytrade/ +├── plugins/ +│ ├── harbor-hub/ +│ └── tastytrade/ +└── skills/ + └── parallel-agent-isolation/ ``` Plugins live under `plugins/`, one directory each, named for the platform they talk to rather than for being an MCP server. Skills and other components get their own top-level directories as they arrive. diff --git a/skills/parallel-agent-isolation/Makefile b/skills/parallel-agent-isolation/Makefile new file mode 100644 index 0000000..8a78f43 --- /dev/null +++ b/skills/parallel-agent-isolation/Makefile @@ -0,0 +1,9 @@ +.PHONY: check evals + +# Packaging and wiring. No API key, no cost. +check: + python3 evals/check_wiring.py + +# Behavioural eval. Needs Claude Code on PATH and working credentials. +evals: + python3 evals/run_evals.py diff --git a/skills/parallel-agent-isolation/SKILL.md b/skills/parallel-agent-isolation/SKILL.md new file mode 100644 index 0000000..9adafb8 --- /dev/null +++ b/skills/parallel-agent-isolation/SKILL.md @@ -0,0 +1,69 @@ +--- +name: parallel-agent-isolation +description: Decide what stateful resources concurrent agents will share, then isolate, serialise, or re-verify serially before dispatching them. Worktrees isolate files, not services, so parallel agents collide on one database or port and both still report green. +when_to_use: Use when two or more agents will run concurrently AND their work touches something stateful outside the filesystem, such as a database, a docker compose stack, an integration or end-to-end suite, a search index, a message broker, a cache, a fixed port, a shared cloud resource or IaC state file, or a shared test account. Also use when judging a finished parallel agent's report, especially one containing "port is already allocated", "address already in use", "container name is already in use", or any statement that the agent reused services it could not start itself. Do not use when concurrent agents only read and write files, such as editing docs or independent source modules, since worktrees already isolate that. +--- + +# Parallel agents share more than the filesystem + +Ask this before dispatch: + +> **What stateful thing outside the filesystem will these agents share?** + +A git worktree isolates files. It isolates nothing else. Two agents in separate +worktrees still run their suites against the one Postgres listening on 5433. + +## What counts + +- Databases, and the fixtures that truncate or seed them +- Search indexes, message brokers, caches +- Fixed ports, and docker compose project names derived from a directory name +- Shared cloud resources: a bucket, a queue, a deployed stack, one IaC state file +- A shared test account, or an API key with per-account state or rate limits + +If nothing on that list is in play, the work is file-local. Dispatch in parallel +and stop reading. + +## Pick one + +| Strategy | Fits when | +| --- | --- | +| **Isolate.** Each agent gets its own instance on its own ports and its own compose project name. | The stack is cheap to duplicate and the parallel work is long enough to repay the setup. | +| **Serialise the stateful step.** Agents work in parallel; a lock, or the dispatcher, lets one run the suite at a time. | The shared step is short next to the work around it. | +| **Parallel, then re-verify serially.** Let them run, treat every agent's test result as unverified, and re-run the suites yourself one at a time. | The suite is fast and the agents' edits do not conflict. Say up front that their results are advisory. | +| **Split the work.** One agent owns everything that touches the resource; the rest parallelise around it. | Only some of the tasks need the service at all. | + +Re-verify serially when unsure. It is the cheapest of the four to get right, and +the only one that still works when an agent ignores its instructions. + +## The collision is silent + +Two suites on one database interleave. Fixtures that truncate tables around each +test delete the other run's rows mid-test. Both suites go green. Both reports +read exactly like reports from runs that proved something. + +So an agent that could not start its own services and used the ones already +running has reported a **corrupted test run, not a workaround**. Any of these in +an agent's report invalidates that agent's verification: + +- `port is already allocated`, `address already in use` +- `container name ... is already in use` +- "the services were already up", "reused the existing stack", "skipped `docker compose up`" + +## Make it detectable + +Put the reporting requirement in the dispatch prompt, not just the expectation: + +> Report how you ran the tests: the exact command, whether you started your own +> services, and the ports and compose project you used. If you could not start +> your own services, stop and say so rather than using the ones already running. + +A green result that does not say how it was produced is unverified. + +## Clean up + +- Containers stranded in `created` or `exited` by a failed start: `docker ps -a`, + then remove by compose project. +- Worktrees: `git worktree list`, then `git worktree remove`. +- Branches a worktree pinned. The branch will not delete while its worktree + exists, so remove worktrees first and branches second. diff --git a/skills/parallel-agent-isolation/evals/README.md b/skills/parallel-agent-isolation/evals/README.md new file mode 100644 index 0000000..31e5164 --- /dev/null +++ b/skills/parallel-agent-isolation/evals/README.md @@ -0,0 +1,56 @@ +# parallel-agent-isolation evals + +Two checks, in cost order. + +```bash +make check # packaging and wiring, no API key, no cost +make evals # behavioural, needs Claude Code on PATH and credentials +``` + +## `make check` + +`check_wiring.py` reads the skill's frontmatter, its marketplace entry, and the +case set. It exists because every way a skill breaks in packaging is silent: an +unclosed frontmatter fence, a `name` that no longer matches the directory, a +marketplace entry whose `skills` path points somewhere else, a `version` pinned +in the entry so installed copies never refresh. In each case the skill installs +and simply never loads. Stdlib only, so it runs anywhere. + +## `make evals` + +`run_evals.py` runs each case as headless Claude Code in a throwaway workspace +holding this skill and nothing else, with `--setting-sources project` so a +personal skill on the developer's machine cannot stand in for the one under +test. Every case asserts on skill invocation; the triggering cases additionally +grade the answer against a rubric using a second, tool-less Claude Code call +that returns a structured verdict and never sees the skill. + +| Case | Skill must | The answer must | +| --- | --- | --- | +| `shared-database-dispatch` | load | resolve the shared Postgres and OpenSearch by isolating, serialising, re-verifying serially, or giving the stateful work to one agent, rather than dispatching both agents at the suite concurrently | +| `green-report-collision` | load | treat a green suite reported alongside `port is already allocated` as invalid, extend the doubt to the other agent's green result, and re-run serially | +| `file-only-parallel` | stay out | (not graded) | + +`file-only-parallel` is the honesty check, and it earns its place. A description +that fires on every mention of parallel agents would pass the other two cases +while making the skill noise, so one case dispatches three agents over +documentation edits and requires the skill to stay out of it. The first draft of +the description failed exactly there, and so did a later attempt to shorten the +clause that excludes file-only work. + +Whether a description triggers is sampled behaviour, so each case runs three +times and every run must hold. Cost is roughly $1 a full run on `sonnet`, capped +per call by `--max-budget-usd`. `EVAL_RUNS`, `EVAL_MODEL`, +`EVAL_MAX_BUDGET_USD`, and `EVAL_TIMEOUT_SEC` override the defaults. Pass case +names as arguments to run a subset. + +## Why not the Harbor harness + +The [harbor-hub evals](../../../plugins/harbor-hub/evals/) are Harbor tasks: a +container image, an MCP server declared in `task.toml`, and a verifier that +recomputes ground truth from the hub. That shape fits a capability surface with +a live backend to compare against. This skill has no server, no backend, and +nothing to recompute. What it has is a claim about what a model does at dispatch +time, so the eval runs the model and looks. Wrapping that in Harbor would add an +image build, a Modal or Docker dependency, and a Harbor API key to a check whose +whole content is one prompt and one grading rubric. diff --git a/skills/parallel-agent-isolation/evals/cases/file-only-parallel/case.json b/skills/parallel-agent-isolation/evals/cases/file-only-parallel/case.json new file mode 100644 index 0000000..f68724d --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/file-only-parallel/case.json @@ -0,0 +1,4 @@ +{ + "expect_skill": false, + "rubric": null +} diff --git a/skills/parallel-agent-isolation/evals/cases/file-only-parallel/prompt.md b/skills/parallel-agent-isolation/evals/cases/file-only-parallel/prompt.md new file mode 100644 index 0000000..cd2223a --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/file-only-parallel/prompt.md @@ -0,0 +1,7 @@ +I want to dispatch three subagents in parallel over this repo, each in its own +git worktree. One writes the module docstrings for `src/parsing/`, one does the +same for `src/rendering/`, and one rewrites the README's install section. + +None of them run tests or start anything. Write me the dispatch plan. + +Do not run anything. Just give me the plan. diff --git a/skills/parallel-agent-isolation/evals/cases/green-report-collision/case.json b/skills/parallel-agent-isolation/evals/cases/green-report-collision/case.json new file mode 100644 index 0000000..6d565cd --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/green-report-collision/case.json @@ -0,0 +1,8 @@ +{ + "expect_skill": true, + "rubric": [ + "The answer must say the work is not done, and must treat the port collision as invalidating the agent's green test result rather than as a harmless note or a minor warning.", + "The answer must state or clearly imply that both agents ran against the same shared services, so both green results are untrustworthy, not only the one that reported the collision.", + "The answer must call for re-running the suites serially against a known-clean stack before trusting either result." + ] +} diff --git a/skills/parallel-agent-isolation/evals/cases/green-report-collision/prompt.md b/skills/parallel-agent-isolation/evals/cases/green-report-collision/prompt.md new file mode 100644 index 0000000..76e444e --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/green-report-collision/prompt.md @@ -0,0 +1,11 @@ +Two subagents I dispatched in parallel have finished. They worked in separate +git worktrees on the same repo. Here is what the second one sent back: + +> Implemented the migration fix. Ran `make test-integration`: 84 passed, 0 +> failed. Note: `docker compose up -d` printed `Error response from daemon: Bind +> for 0.0.0.0:5433 failed: port is already allocated`, so I used the Postgres +> and OpenSearch instances that were already running. Suite is green. + +The first agent reported green as well. Am I done? Tell me what to do next. + +Do not run anything. Just answer. diff --git a/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/case.json b/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/case.json new file mode 100644 index 0000000..455dbc7 --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/case.json @@ -0,0 +1,8 @@ +{ + "expect_skill": true, + "rubric": [ + "The plan must recognise that the two agents would share the Postgres and OpenSearch services even though each has its own git worktree, and must not dispatch both agents to run the integration suite concurrently against the one shared stack.", + "The plan must resolve that sharing in at least one of these ways: give each agent its own service instance on its own ports or compose project; serialise the integration-suite step so only one agent runs it at a time; keep the agents parallel but re-run the suites serially afterwards and treat the agents' own results as unverified; or give the integration work to a single agent.", + "It is not enough to merely mention worktrees, or to say the agents should 'be careful' or 'coordinate' without naming one of those resolutions." + ] +} diff --git a/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/prompt.md b/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/prompt.md new file mode 100644 index 0000000..a8079b4 --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/prompt.md @@ -0,0 +1,13 @@ +I maintain a Python service. Its integration suite talks to Postgres on port +5433 and OpenSearch on port 9200, both brought up by `docker compose up -d` from +the repo root. The unit suite needs neither. + +I have two independent pieces of work queued: adding a new retrieval strategy, +and fixing a migration ordering bug. Each one needs `make test-integration` to +pass before I will look at it. + +I want to hand both to subagents at the same time, each in its own git worktree, +and have them report back. Write me the dispatch plan: how many agents, what +each one does, and what I put in their prompts. + +Do not run anything. Just give me the plan. diff --git a/skills/parallel-agent-isolation/evals/check_wiring.py b/skills/parallel-agent-isolation/evals/check_wiring.py new file mode 100644 index 0000000..a4686e6 --- /dev/null +++ b/skills/parallel-agent-isolation/evals/check_wiring.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Structural checks for the parallel-agent-isolation skill. + +A skill fails silently when its packaging is wrong: an unparsed frontmatter +block, a name that no longer matches the directory, or a marketplace entry that +does not point at the skill directory all leave the skill installed and never +loaded. These checks run without an API key so they can gate every PR. + +Stdlib only, and the frontmatter is parsed by hand, because a top-level skill +directory has no dependency manifest to hang PyYAML off. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +SKILL_DIR = Path(__file__).resolve().parent.parent +REPO_ROOT = SKILL_DIR.parent.parent +SKILL_NAME = SKILL_DIR.name + +# Claude Code truncates the combined description and when_to_use text at this +# many characters in the skill listing, so anything past it never reaches the +# model that decides whether to load the skill. +LISTING_CAP = 1536 + +failures: list[str] = [] + + +def check(condition: bool, message: str) -> bool: + if not condition: + failures.append(message) + return condition + + +def parse_frontmatter(text: str) -> dict[str, str]: + """Read a `key: value` frontmatter block, one entry per line.""" + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + failures.append("SKILL.md does not open with a --- frontmatter fence") + return {} + try: + end = lines.index("---", 1) + except ValueError: + failures.append("SKILL.md frontmatter is never closed with ---") + return {} + + fields: dict[str, str] = {} + for lineno, line in enumerate(lines[1:end], start=2): + if not line.strip(): + continue + key, sep, value = line.partition(":") + if not sep or not key.strip() or key.startswith((" ", "\t")): + failures.append( + f"SKILL.md line {lineno} is not a single-line `key: value` entry: {line!r}" + ) + continue + fields[key.strip()] = value.strip() + return fields + + +def check_skill_md() -> None: + path = SKILL_DIR / "SKILL.md" + if not check(path.is_file(), f"missing {path}"): + return + fields = parse_frontmatter(path.read_text()) + + check( + fields.get("name") == SKILL_NAME, + f"frontmatter name is {fields.get('name')!r}, expected the directory name {SKILL_NAME!r}", + ) + check( + bool(fields.get("description")), + "frontmatter has no description, so nothing tells Claude when to load the skill", + ) + + listing = f"{fields.get('description', '')} {fields.get('when_to_use', '')}".strip() + check( + len(listing) <= LISTING_CAP, + f"description plus when_to_use is {len(listing)} characters and is truncated at {LISTING_CAP}", + ) + + body = path.read_text().split("---", 2)[-1] + # The detection signal is the part of the skill that is easiest to lose in an + # edit and the hardest to notice missing, since the failure it names is silent. + for phrase in ("port is already allocated", "already in use"): + check(phrase in body, f"the body no longer names the collision signal {phrase!r}") + + +def check_marketplace_entry() -> None: + path = REPO_ROOT / ".claude-plugin" / "marketplace.json" + if not check(path.is_file(), f"missing {path}"): + return + manifest = json.loads(path.read_text()) + + entries = [p for p in manifest.get("plugins", []) if p.get("name") == SKILL_NAME] + if not check( + len(entries) == 1, + f"expected exactly one marketplace entry named {SKILL_NAME!r}, found {len(entries)}", + ): + return + entry = entries[0] + + check( + entry.get("source") == "./", + f"entry source is {entry.get('source')!r}, expected './' for a marketplace-root skill", + ) + check( + entry.get("strict") is False, + "entry must set strict to false: there is no plugin.json at the repository root to be the authority", + ) + check( + entry.get("skills") == [f"./skills/{SKILL_NAME}"], + f"entry skills is {entry.get('skills')!r}, expected ['./skills/{SKILL_NAME}']", + ) + check( + bool(entry.get("description")), + "entry has no description, so the plugin listing shows nothing", + ) + check( + "version" not in entry, + "entry pins a version; omit it so the commit SHA is the version and installed copies refresh on every change", + ) + + +def check_cases() -> None: + cases_dir = SKILL_DIR / "evals" / "cases" + cases = sorted(p for p in cases_dir.iterdir() if p.is_dir()) if cases_dir.is_dir() else [] + if not check(len(cases) >= 2, f"expected at least two eval cases in {cases_dir}"): + return + + expectations = set() + for case in cases: + if not check((case / "prompt.md").is_file(), f"{case.name}: missing prompt.md"): + continue + spec_path = case / "case.json" + if not check(spec_path.is_file(), f"{case.name}: missing case.json"): + continue + spec = json.loads(spec_path.read_text()) + if check( + isinstance(spec.get("expect_skill"), bool), + f"{case.name}: expect_skill must be a boolean", + ): + expectations.add(spec["expect_skill"]) + if spec["expect_skill"]: + check( + bool(spec.get("rubric")), + f"{case.name}: a triggering case needs a rubric to grade the answer against", + ) + + check( + expectations == {True, False}, + "the case set needs both a case the skill must load for and one it must stay out of; a description that " + "triggers on everything is not a working description", + ) + + +check_skill_md() +check_marketplace_entry() +check_cases() + +for failure in failures: + print(f"FAIL: {failure}", file=sys.stderr) +if failures: + sys.exit(1) +print(f"ok: {SKILL_NAME} packaging, marketplace entry, and eval cases") diff --git a/skills/parallel-agent-isolation/evals/run_evals.py b/skills/parallel-agent-isolation/evals/run_evals.py new file mode 100644 index 0000000..25d4d1e --- /dev/null +++ b/skills/parallel-agent-isolation/evals/run_evals.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Behavioural eval for the parallel-agent-isolation skill. + +Each case runs headless Claude Code in a throwaway workspace that contains only +this skill, and checks two things: whether the skill loaded on its own, and +whether the answer resolves the shared-resource problem instead of dispatching +naively. A rubric is graded by a second, tool-less Claude Code call returning a +structured verdict. + +Both halves matter. A skill that never loads is inert whatever its body says, +and a skill that loads for every parallel dispatch is noise, so one case must +not trigger it at all. + +Usage: python3 run_evals.py [case-name ...] +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +SKILL_DIR = Path(__file__).resolve().parent.parent +SKILL_NAME = SKILL_DIR.name +CASES_DIR = Path(__file__).resolve().parent / "cases" + +MODEL = os.environ.get("EVAL_MODEL", "sonnet") +BUDGET_USD = os.environ.get("EVAL_MAX_BUDGET_USD", "1.00") +TIMEOUT_SEC = int(os.environ.get("EVAL_TIMEOUT_SEC", "600")) +# Whether a description triggers is a sampled behaviour, so one run of a case +# says little. Every run of a case must hold for the case to pass. +RUNS = int(os.environ.get("EVAL_RUNS", "3")) + +VERDICT_SCHEMA = { + "type": "object", + "properties": { + "verdict": {"type": "string", "enum": ["pass", "fail"]}, + "reason": {"type": "string"}, + }, + "required": ["verdict", "reason"], +} + + +class EvalError(RuntimeError): + pass + + +def run_claude(prompt: str, cwd: Path, extra: list[str]) -> list[dict]: + """Run headless Claude Code and return the parsed stream-json events.""" + command = [ + "claude", + "-p", + prompt, + "--output-format", + "stream-json", + "--verbose", + "--model", + MODEL, + # Only the project's own .claude/ is loaded, so a personal skill on the + # developer's machine cannot stand in for the one under test. + "--setting-sources", + "project", + "--strict-mcp-config", + "--no-session-persistence", + "--max-budget-usd", + BUDGET_USD, + *extra, + ] + result = subprocess.run(command, cwd=cwd, capture_output=True, text=True, timeout=TIMEOUT_SEC) + if result.returncode != 0: + raise EvalError(f"claude exited {result.returncode}: {result.stderr.strip()[:500]}") + + events = [json.loads(line) for line in result.stdout.splitlines() if line.strip()] + if not events: + raise EvalError("claude produced no output") + return events + + +def final_text(events: list[dict]) -> str: + for event in reversed(events): + if event.get("type") == "result": + if event.get("is_error"): + raise EvalError( + f"claude reported an error result: {str(event.get('result'))[:300]}" + ) + return event.get("result") or "" + raise EvalError("no result event in the stream") + + +def total_cost(events: list[dict]) -> float: + for event in reversed(events): + if event.get("type") == "result": + return float(event.get("total_cost_usd") or 0.0) + return 0.0 + + +def skill_was_loaded(events: list[dict]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + for block in event.get("message", {}).get("content", []): + if block.get("type") == "tool_use" and block.get("name") == "Skill": + if (block.get("input") or {}).get("skill") == SKILL_NAME: + return True + return False + + +def judge(rubric: list[str], answer: str, workspace: Path) -> tuple[bool, str]: + """Grade an answer against a rubric with a tool-less Claude Code call.""" + criteria = "\n".join(f"{i}. {line}" for i, line in enumerate(rubric, start=1)) + prompt = ( + "You are grading one answer against fixed criteria. Judge only what the answer says.\n" + "Return verdict 'pass' only if every criterion is met, otherwise 'fail'.\n" + "Keep reason to one sentence, and quote the answer where it decides the verdict.\n\n" + f"CRITERIA\n{criteria}\n\nANSWER\n{answer}\n" + ) + events = run_claude( + prompt, + workspace, + ["--tools", "", "--json-schema", json.dumps(VERDICT_SCHEMA)], + ) + try: + parsed = json.loads(final_text(events)) + except json.JSONDecodeError as exc: + raise EvalError(f"judge did not return the verdict schema: {exc}") from exc + # The judge occasionally trails a stray closing tag after its reason. + reason = re.sub(r"(\s*)+\s*$", "", " ".join(parsed["reason"].split())) + return parsed["verdict"] == "pass", reason + + +def run_case(case: Path, workspace: Path, judge_workspace: Path) -> tuple[bool, str, float]: + """Run one case RUNS times. Every run must hold, so one lucky pass is not a pass.""" + spec = json.loads((case / "case.json").read_text()) + prompt = (case / "prompt.md").read_text() + cost = 0.0 + + for attempt in range(1, RUNS + 1): + events = run_claude(prompt, workspace, ["--tools", "Skill"]) + cost += total_cost(events) + + loaded = skill_was_loaded(events) + if loaded != spec["expect_skill"]: + want = "load" if spec["expect_skill"] else "stay out of" + return False, f"run {attempt}/{RUNS}: the skill did not {want} this scenario", cost + + if not spec.get("rubric"): + continue + + passed, reason = judge(spec["rubric"], final_text(events), judge_workspace) + if not passed: + return False, f"run {attempt}/{RUNS}: {reason}", cost + + return True, f"{RUNS}/{RUNS} runs held", cost + + +def main() -> int: + if not shutil.which("claude"): + print("claude is not on PATH; install Claude Code to run these evals.", file=sys.stderr) + return 1 + + wanted = set(sys.argv[1:]) + cases = sorted( + p for p in CASES_DIR.iterdir() if p.is_dir() and (not wanted or p.name in wanted) + ) + if not cases: + print(f"no cases matched {sorted(wanted)}", file=sys.stderr) + return 1 + + with tempfile.TemporaryDirectory(prefix=f"{SKILL_NAME}-evals-") as tmp: + root = Path(tmp) + # The agent workspace holds this skill and nothing else, so a load is + # attributable to this skill's description rather than to the repository. + workspace = root / "workspace" + shutil.copytree( + SKILL_DIR, + workspace / ".claude" / "skills" / SKILL_NAME, + ignore=shutil.ignore_patterns("evals"), + ) + # The judge gets an empty workspace: it must not see the skill it grades against. + judge_workspace = root / "judge" + judge_workspace.mkdir() + + failed: list[str] = [] + spent = 0.0 + for case in cases: + try: + passed, reason, cost = run_case(case, workspace, judge_workspace) + except (EvalError, subprocess.TimeoutExpired) as exc: + passed, reason, cost = False, str(exc), 0.0 + spent += cost + print(f"{'PASS' if passed else 'FAIL'} {case.name}: {reason}", flush=True) + if not passed: + failed.append(case.name) + + print(f"\n{len(cases) - len(failed)}/{len(cases)} cases passed, ${spent:.2f} spent") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) From c8b37c1164b05d14d3b16122d9238ecbf3d19d51 Mon Sep 17 00:00:00 2001 From: Walker Hughes <74113220+walkerhughes@users.noreply.github.com> Date: Sun, 2 Aug 2026 18:44:33 -0700 Subject: [PATCH 2/3] refactor(skills): generalise parallel-agent-isolation past its origin stack The skill was written from one session on a Python service and carried that session's details as though they were the universe: a Postgres on a specific nonstandard port, a search index on another, docker compose as the only way services start. The idea underneath is general. Concurrent agents collide on stateful resources that filesystem isolation does not cover, and the collision is silent because both runs still report green. That is as true of a shared staging database, one booted simulator, one remote state file, or one API account as it is of a container stack. Substitute rather than abstract. The examples stay concrete, but each is now one instance among several drawn from different ecosystems, so a reader on a stack the skill does not name still recognises the situation. The collision signals keep the container error strings as examples and name the class they belong to: any claim that the agent reused a resource it could not create. Cleanup frames stranded containers as the common case of what an aborted start leaves behind, next to a booted device, a held lock, a half-applied stack. The case set is the part that had to prove this rather than assert it. Every triggering case was a container-and-database scenario, so a skill that only recognised containers and databases would have passed. The cases now span a Go service against one hosted staging database, a Terraform repo with one remote state file and one sandbox account, and an iOS app with one booted simulator, where the collision signal is not a container error string at all. file-only-parallel stays as the negative control, and it still holds against the widened description: it must not load for three agents editing docs. All four cases pass three runs of three, $1.31. --- skills/parallel-agent-isolation/SKILL.md | 49 ++++++++++--------- .../parallel-agent-isolation/evals/README.md | 21 +++++--- .../cases/green-report-collision/case.json | 6 +-- .../cases/green-report-collision/prompt.md | 10 ++-- .../cases/shared-database-dispatch/case.json | 4 +- .../cases/shared-database-dispatch/prompt.md | 13 ++--- .../evals/cases/shared-infra-state/case.json | 8 +++ .../evals/cases/shared-infra-state/prompt.md | 14 ++++++ 8 files changed, 79 insertions(+), 46 deletions(-) create mode 100644 skills/parallel-agent-isolation/evals/cases/shared-infra-state/case.json create mode 100644 skills/parallel-agent-isolation/evals/cases/shared-infra-state/prompt.md diff --git a/skills/parallel-agent-isolation/SKILL.md b/skills/parallel-agent-isolation/SKILL.md index 9adafb8..e161c2a 100644 --- a/skills/parallel-agent-isolation/SKILL.md +++ b/skills/parallel-agent-isolation/SKILL.md @@ -1,7 +1,7 @@ --- name: parallel-agent-isolation -description: Decide what stateful resources concurrent agents will share, then isolate, serialise, or re-verify serially before dispatching them. Worktrees isolate files, not services, so parallel agents collide on one database or port and both still report green. -when_to_use: Use when two or more agents will run concurrently AND their work touches something stateful outside the filesystem, such as a database, a docker compose stack, an integration or end-to-end suite, a search index, a message broker, a cache, a fixed port, a shared cloud resource or IaC state file, or a shared test account. Also use when judging a finished parallel agent's report, especially one containing "port is already allocated", "address already in use", "container name is already in use", or any statement that the agent reused services it could not start itself. Do not use when concurrent agents only read and write files, such as editing docs or independent source modules, since worktrees already isolate that. +description: Decide what stateful resources concurrent agents will share, then isolate, serialise, or re-verify serially before dispatching them. Worktrees isolate files, not services, so parallel agents collide on one database, one simulator, or one state file and both still report green. +when_to_use: Use when two or more agents will run concurrently AND their work touches something stateful outside the filesystem, such as a database, a service stack, an integration or end-to-end suite, a search index, a message broker, a cache, a fixed port, a device or simulator, a shared cloud environment or IaC state file, or a shared account or API key. Also use when judging a finished parallel agent's report, especially one saying it could not get its own instance of such a resource and used the one already running, in any wording, including "port is already allocated", "address already in use", "container name is already in use", or "no free simulator". Do not use when concurrent agents only read and write files, such as editing docs or independent source modules, since worktrees already isolate that. --- # Parallel agents share more than the filesystem @@ -11,14 +11,16 @@ Ask this before dispatch: > **What stateful thing outside the filesystem will these agents share?** A git worktree isolates files. It isolates nothing else. Two agents in separate -worktrees still run their suites against the one Postgres listening on 5433. +worktrees still query the one database, boot the one simulator, and lock the one +remote state file. ## What counts -- Databases, and the fixtures that truncate or seed them -- Search indexes, message brokers, caches -- Fixed ports, and docker compose project names derived from a directory name -- Shared cloud resources: a bucket, a queue, a deployed stack, one IaC state file +- Databases and search indexes, and the fixtures that truncate or seed them +- Message brokers and caches, and anything holding state between requests +- Fixed ports, and container or project names derived from a directory name +- Devices, emulators, and simulators, where one is booted at a time +- Shared cloud resources: a bucket, a queue, a staging environment, an IaC state file - A shared test account, or an API key with per-account state or rate limits If nothing on that list is in play, the work is file-local. Dispatch in parallel @@ -28,10 +30,10 @@ and stop reading. | Strategy | Fits when | | --- | --- | -| **Isolate.** Each agent gets its own instance on its own ports and its own compose project name. | The stack is cheap to duplicate and the parallel work is long enough to repay the setup. | +| **Isolate.** Each agent gets its own instance, on its own ports, schema, namespace, or account. | The resource is cheap to duplicate and the parallel work is long enough to repay the setup. | | **Serialise the stateful step.** Agents work in parallel; a lock, or the dispatcher, lets one run the suite at a time. | The shared step is short next to the work around it. | | **Parallel, then re-verify serially.** Let them run, treat every agent's test result as unverified, and re-run the suites yourself one at a time. | The suite is fast and the agents' edits do not conflict. Say up front that their results are advisory. | -| **Split the work.** One agent owns everything that touches the resource; the rest parallelise around it. | Only some of the tasks need the service at all. | +| **Split the work.** One agent owns everything that touches the resource; the rest parallelise around it. | Only some of the tasks need the resource at all. | Re-verify serially when unsure. It is the cheapest of the four to get right, and the only one that still works when an agent ignores its instructions. @@ -39,31 +41,34 @@ the only one that still works when an agent ignores its instructions. ## The collision is silent Two suites on one database interleave. Fixtures that truncate tables around each -test delete the other run's rows mid-test. Both suites go green. Both reports -read exactly like reports from runs that proved something. +test delete the other run's rows mid-test. Both suites go green. The shape +repeats wherever state is shared: two runs driving one simulator tap each +other's screens, two applies against one state file each plan against a world +the other has already changed. -So an agent that could not start its own services and used the ones already -running has reported a **corrupted test run, not a workaround**. Any of these in -an agent's report invalidates that agent's verification: +So an agent that could not get its own instance and used the one already there +has reported a **corrupted run, not a workaround**. The signal is any claim that +the agent reused a resource it could not create, whatever the wording: -- `port is already allocated`, `address already in use` -- `container name ... is already in use` -- "the services were already up", "reused the existing stack", "skipped `docker compose up`" +- `port is already allocated`, `address already in use`, `container name ... is already in use` +- "no free simulator, so I used the booted one", "the shared staging database was already migrated" +- "the services were already up", "reused the existing stack", "skipped the setup step" ## Make it detectable Put the reporting requirement in the dispatch prompt, not just the expectation: -> Report how you ran the tests: the exact command, whether you started your own -> services, and the ports and compose project you used. If you could not start -> your own services, stop and say so rather than using the ones already running. +> Report how you ran the tests: the exact command, and whether every service, +> device, environment, and account it touched was yours alone. If you could not +> get your own, stop and say so rather than using the one already running. A green result that does not say how it was produced is unverified. ## Clean up -- Containers stranded in `created` or `exited` by a failed start: `docker ps -a`, - then remove by compose project. +- Whatever an aborted start leaves half-made: a booted device, a held lock, a + partly applied stack. Containers are the common case, stranded in `created` or + `exited`: `docker ps -a`, then remove by project. - Worktrees: `git worktree list`, then `git worktree remove`. - Branches a worktree pinned. The branch will not delete while its worktree exists, so remove worktrees first and branches second. diff --git a/skills/parallel-agent-isolation/evals/README.md b/skills/parallel-agent-isolation/evals/README.md index 31e5164..ed0db62 100644 --- a/skills/parallel-agent-isolation/evals/README.md +++ b/skills/parallel-agent-isolation/evals/README.md @@ -25,22 +25,27 @@ test. Every case asserts on skill invocation; the triggering cases additionally grade the answer against a rubric using a second, tool-less Claude Code call that returns a structured verdict and never sees the skill. -| Case | Skill must | The answer must | -| --- | --- | --- | -| `shared-database-dispatch` | load | resolve the shared Postgres and OpenSearch by isolating, serialising, re-verifying serially, or giving the stateful work to one agent, rather than dispatching both agents at the suite concurrently | -| `green-report-collision` | load | treat a green suite reported alongside `port is already allocated` as invalid, extend the doubt to the other agent's green result, and re-run serially | -| `file-only-parallel` | stay out | (not graded) | +The triggering cases deliberately sit in different ecosystems and share different +kinds of state, because a skill that recognised only containers and databases +would pass a case set drawn from one stack while being useless on the next one. + +| Case | Stack, and what is shared | Skill must | The answer must | +| --- | --- | --- | --- | +| `shared-database-dispatch` | Go, one hosted staging database | load | resolve the shared instance by isolating, serialising, re-verifying serially, or giving the stateful work to one agent, rather than dispatching both agents at the suite concurrently | +| `shared-infra-state` | Terraform, one remote state file and one sandbox account | load | resolve the shared state and account the same way, rather than letting three applies race through the lock | +| `green-report-collision` | iOS, one booted simulator | load | treat a green suite reported alongside "a second simulator would not boot" as invalid, extend the doubt to the other agent's green result, and re-run serially | +| `file-only-parallel` | any, nothing shared | stay out | (not graded) | `file-only-parallel` is the honesty check, and it earns its place. A description -that fires on every mention of parallel agents would pass the other two cases +that fires on every mention of parallel agents would pass the triggering cases while making the skill noise, so one case dispatches three agents over documentation edits and requires the skill to stay out of it. The first draft of the description failed exactly there, and so did a later attempt to shorten the clause that excludes file-only work. Whether a description triggers is sampled behaviour, so each case runs three -times and every run must hold. Cost is roughly $1 a full run on `sonnet`, capped -per call by `--max-budget-usd`. `EVAL_RUNS`, `EVAL_MODEL`, +times and every run must hold. Cost is a dollar or so a full run on `sonnet`, +capped per call by `--max-budget-usd`. `EVAL_RUNS`, `EVAL_MODEL`, `EVAL_MAX_BUDGET_USD`, and `EVAL_TIMEOUT_SEC` override the defaults. Pass case names as arguments to run a subset. diff --git a/skills/parallel-agent-isolation/evals/cases/green-report-collision/case.json b/skills/parallel-agent-isolation/evals/cases/green-report-collision/case.json index 6d565cd..778e8b5 100644 --- a/skills/parallel-agent-isolation/evals/cases/green-report-collision/case.json +++ b/skills/parallel-agent-isolation/evals/cases/green-report-collision/case.json @@ -1,8 +1,8 @@ { "expect_skill": true, "rubric": [ - "The answer must say the work is not done, and must treat the port collision as invalidating the agent's green test result rather than as a harmless note or a minor warning.", - "The answer must state or clearly imply that both agents ran against the same shared services, so both green results are untrustworthy, not only the one that reported the collision.", - "The answer must call for re-running the suites serially against a known-clean stack before trusting either result." + "The answer must say the work is not done, and must treat the reuse of the already-booted simulator as invalidating the agent's green test result rather than as a harmless note or a minor warning.", + "The answer must state or clearly imply that both agents ran against the same simulator, so both green results are untrustworthy, not only the one that reported the problem.", + "The answer must call for re-running the suites serially against a known-clean simulator before trusting either result." ] } diff --git a/skills/parallel-agent-isolation/evals/cases/green-report-collision/prompt.md b/skills/parallel-agent-isolation/evals/cases/green-report-collision/prompt.md index 76e444e..0d90cdf 100644 --- a/skills/parallel-agent-isolation/evals/cases/green-report-collision/prompt.md +++ b/skills/parallel-agent-isolation/evals/cases/green-report-collision/prompt.md @@ -1,10 +1,10 @@ Two subagents I dispatched in parallel have finished. They worked in separate -git worktrees on the same repo. Here is what the second one sent back: +git worktrees on the same iOS app. Here is what the second one sent back: -> Implemented the migration fix. Ran `make test-integration`: 84 passed, 0 -> failed. Note: `docker compose up -d` printed `Error response from daemon: Bind -> for 0.0.0.0:5433 failed: port is already allocated`, so I used the Postgres -> and OpenSearch instances that were already running. Suite is green. +> Fixed the checkout crash. Ran the UI suite with `xcodebuild test`: 84 passed, +> 0 failed. Note: a second simulator would not boot, so I ran against the +> iPhone 16 simulator that was already booted and installed over the build that +> was on it. Suite is green. The first agent reported green as well. Am I done? Tell me what to do next. diff --git a/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/case.json b/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/case.json index 455dbc7..2ec97ee 100644 --- a/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/case.json +++ b/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/case.json @@ -1,8 +1,8 @@ { "expect_skill": true, "rubric": [ - "The plan must recognise that the two agents would share the Postgres and OpenSearch services even though each has its own git worktree, and must not dispatch both agents to run the integration suite concurrently against the one shared stack.", - "The plan must resolve that sharing in at least one of these ways: give each agent its own service instance on its own ports or compose project; serialise the integration-suite step so only one agent runs it at a time; keep the agents parallel but re-run the suites serially afterwards and treat the agents' own results as unverified; or give the integration work to a single agent.", + "The plan must recognise that the two agents would share the one staging Postgres instance even though each has its own git worktree, and must not dispatch both agents to run the e2e suite concurrently against it.", + "The plan must resolve that sharing in at least one of these ways: give each agent its own database instance, schema, or namespace; serialise the e2e step so only one agent runs it at a time; keep the agents parallel but re-run the suites serially afterwards and treat the agents' own results as unverified; or give the e2e work to a single agent.", "It is not enough to merely mention worktrees, or to say the agents should 'be careful' or 'coordinate' without naming one of those resolutions." ] } diff --git a/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/prompt.md b/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/prompt.md index a8079b4..cadeb0c 100644 --- a/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/prompt.md +++ b/skills/parallel-agent-isolation/evals/cases/shared-database-dispatch/prompt.md @@ -1,10 +1,11 @@ -I maintain a Python service. Its integration suite talks to Postgres on port -5433 and OpenSearch on port 9200, both brought up by `docker compose up -d` from -the repo root. The unit suite needs neither. +I maintain a Go service. Its end-to-end tests run with `go test -tags e2e ./...` +against our shared staging Postgres, a managed instance in our cloud account +that everyone on the team points at. There is no local copy of it. The unit +tests need nothing. -I have two independent pieces of work queued: adding a new retrieval strategy, -and fixing a migration ordering bug. Each one needs `make test-integration` to -pass before I will look at it. +I have two independent pieces of work queued: adding pagination to the orders +endpoint, and fixing a timezone bug in the reporting query. Each one needs the +e2e suite to pass before I will look at it. I want to hand both to subagents at the same time, each in its own git worktree, and have them report back. Write me the dispatch plan: how many agents, what diff --git a/skills/parallel-agent-isolation/evals/cases/shared-infra-state/case.json b/skills/parallel-agent-isolation/evals/cases/shared-infra-state/case.json new file mode 100644 index 0000000..f3a62c2 --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/shared-infra-state/case.json @@ -0,0 +1,8 @@ +{ + "expect_skill": true, + "rubric": [ + "The plan must recognise that the three agents would share the one Terraform state file and the one sandbox account even though each has its own git worktree, and must not dispatch all three to apply concurrently against that single state and account.", + "The plan must resolve that sharing in at least one of these ways: give each agent its own state, workspace, or sandbox account so the applies do not meet; serialise the apply and smoke-test step so only one agent runs it at a time; keep the agents parallel on the code but do the applies and verification serially afterwards and treat the agents' own results as unverified; or give all the applying to a single agent.", + "It is not enough to merely mention worktrees, or to rely on the state lock alone, or to say the agents should 'be careful' or 'coordinate' without naming one of those resolutions." + ] +} diff --git a/skills/parallel-agent-isolation/evals/cases/shared-infra-state/prompt.md b/skills/parallel-agent-isolation/evals/cases/shared-infra-state/prompt.md new file mode 100644 index 0000000..1681b4e --- /dev/null +++ b/skills/parallel-agent-isolation/evals/cases/shared-infra-state/prompt.md @@ -0,0 +1,14 @@ +Our infrastructure is a Terraform repo. The backend is one S3 bucket with a +DynamoDB lock table, and everything applies into a single sandbox cloud account +we use to try changes before production. Verifying a change means applying it in +the sandbox and then running a smoke script against the deployed endpoint. + +Three unrelated changes are queued: a new CDN cache policy, a database instance +resize, and a tightened security group. I want three subagents in parallel, each +in its own git worktree, each applying and smoke-testing its own change and +reporting back. + +Write me the dispatch plan: how many agents, what each one does, and what I put +in their prompts. + +Do not run anything. Just give me the plan. From dcf979394b5250986071ff2feafc052a44517610 Mon Sep 17 00:00:00 2001 From: Walker Hughes <74113220+walkerhughes@users.noreply.github.com> Date: Sun, 2 Aug 2026 23:27:19 -0700 Subject: [PATCH 3/3] refactor(skills): state parallel-agent-isolation as principle and workflow The skill read as an inventory and a runbook. A six-item catalogue of what counts as shared state, a four-row strategy table, a list of collision error strings, a paste-in dispatch prompt, and a cleanup section naming the commands to list containers and worktrees. All of it detail, none of it the idea. The idea is that agents running concurrently share more than the filesystem, so what they will contend for is decided before dispatch, and a result produced under contention is unverified. The body now leads with that, and with the reason it has to be decided rather than diagnosed: the failure is silent. Contending agents do not error, they both report success, and their reports are indistinguishable from honest ones. There is nothing to find afterwards. What survives does so because it changes behaviour where the principle alone would not. The four resolutions stay, as the workflow an orchestrator actually runs, with each one's conditions in prose rather than a table column. The inventory becomes a characterisation: shared state is whatever exists as a single instance and carries changes between calls. The reporting template becomes the principle behind it, that agents must report how they verified and not just the outcome. One anti-pattern is kept whole, the agent that could not get its own instance, used the one already running, and reported green, because it is worth more than the error strings it replaces; those strings stay in when_to_use, which is the trigger surface, not prose. Cleanup is gone. Two things the compression lost, both restored as principles once the evals caught them. Contention is plural, and the obvious resource hides a second, so resolving one leaves the other shared. And a built-in lock covers only the thing it guards, leaving whatever that thing mutates still shared, so the serialised region has to span the change and the verification depending on it. Without those, a plan can serialise the state file, leave the environment it writes to contended, and read as though it had resolved the problem. check_wiring.py asserted that two collision error strings appeared in the body. That check fought a principle-based skill: it pinned prose and would fail the moment the list it named was compressed, pushing the skill back toward being a checklist. It now asserts the body is non-empty, leaving the file's structural assertions to carry the check, frontmatter and marketplace entry and a case set holding both a triggering and a non-triggering case. Verified non-vacuous: an emptied body, a removed negative control, and a wrong marketplace skills path each still fail it. description and when_to_use are unchanged, so the trigger surface that the case set was tuned against is untouched and the marketplace entry needs no edit. All four cases pass three runs of three against the final text, $2.56 across the runs. file-only-parallel, the negative control, was re-run after every change to the body and never loaded the skill. --- skills/parallel-agent-isolation/SKILL.md | 128 +++++++++--------- .../evals/check_wiring.py | 8 +- 2 files changed, 66 insertions(+), 70 deletions(-) diff --git a/skills/parallel-agent-isolation/SKILL.md b/skills/parallel-agent-isolation/SKILL.md index e161c2a..404e804 100644 --- a/skills/parallel-agent-isolation/SKILL.md +++ b/skills/parallel-agent-isolation/SKILL.md @@ -6,69 +6,65 @@ when_to_use: Use when two or more agents will run concurrently AND their work to # Parallel agents share more than the filesystem -Ask this before dispatch: - -> **What stateful thing outside the filesystem will these agents share?** - -A git worktree isolates files. It isolates nothing else. Two agents in separate -worktrees still query the one database, boot the one simulator, and lock the one -remote state file. - -## What counts - -- Databases and search indexes, and the fixtures that truncate or seed them -- Message brokers and caches, and anything holding state between requests -- Fixed ports, and container or project names derived from a directory name -- Devices, emulators, and simulators, where one is booted at a time -- Shared cloud resources: a bucket, a queue, a staging environment, an IaC state file -- A shared test account, or an API key with per-account state or rate limits - -If nothing on that list is in play, the work is file-local. Dispatch in parallel -and stop reading. - -## Pick one - -| Strategy | Fits when | -| --- | --- | -| **Isolate.** Each agent gets its own instance, on its own ports, schema, namespace, or account. | The resource is cheap to duplicate and the parallel work is long enough to repay the setup. | -| **Serialise the stateful step.** Agents work in parallel; a lock, or the dispatcher, lets one run the suite at a time. | The shared step is short next to the work around it. | -| **Parallel, then re-verify serially.** Let them run, treat every agent's test result as unverified, and re-run the suites yourself one at a time. | The suite is fast and the agents' edits do not conflict. Say up front that their results are advisory. | -| **Split the work.** One agent owns everything that touches the resource; the rest parallelise around it. | Only some of the tasks need the resource at all. | - -Re-verify serially when unsure. It is the cheapest of the four to get right, and -the only one that still works when an agent ignores its instructions. - -## The collision is silent - -Two suites on one database interleave. Fixtures that truncate tables around each -test delete the other run's rows mid-test. Both suites go green. The shape -repeats wherever state is shared: two runs driving one simulator tap each -other's screens, two applies against one state file each plan against a world -the other has already changed. - -So an agent that could not get its own instance and used the one already there -has reported a **corrupted run, not a workaround**. The signal is any claim that -the agent reused a resource it could not create, whatever the wording: - -- `port is already allocated`, `address already in use`, `container name ... is already in use` -- "no free simulator, so I used the booted one", "the shared staging database was already migrated" -- "the services were already up", "reused the existing stack", "skipped the setup step" - -## Make it detectable - -Put the reporting requirement in the dispatch prompt, not just the expectation: - -> Report how you ran the tests: the exact command, and whether every service, -> device, environment, and account it touched was yours alone. If you could not -> get your own, stop and say so rather than using the one already running. - -A green result that does not say how it was produced is unverified. - -## Clean up - -- Whatever an aborted start leaves half-made: a booted device, a held lock, a - partly applied stack. Containers are the common case, stranded in `created` or - `exited`: `docker ps -a`, then remove by project. -- Worktrees: `git worktree list`, then `git worktree remove`. -- Branches a worktree pinned. The branch will not delete while its worktree - exists, so remove worktrees first and branches second. +Decide what concurrently running agents will contend for before dispatching +them, and treat any result produced under contention as unverified. + +A git worktree isolates files and nothing else. Agents in separate worktrees +still query the one database, boot the one device, hold the one lock, and spend +the one account's quota. Shared state is whatever exists as a single instance +and carries changes between calls, so what one agent does lands where another +will read it. + +## The failure is silent + +Contention does not raise an error. Two suites against one database interleave, +and a fixture truncating tables between tests deletes the other run's rows +mid-test, so both go green. Two runs driving one device tap each other's +screens. Two applies against one state file each plan against a world the other +has already changed. All of them report success, indistinguishably from an +honest run. + +So this is decided before dispatch, not diagnosed after. Afterwards there is +nothing to find, only the question of whether the result was produced under +contention, and a result produced under contention is unverified whatever it +says. + +## The pre-dispatch workflow + +**1. Name what will be contended for.** Ask what stateful things outside the +filesystem these agents will share, and name each one separately. The obvious +resource usually hides a second, and resolving one leaves the other shared. If +the honest answer is nothing, the work is file-local: dispatch in parallel and +stop here. + +**2. Choose how the contention resolves.** Any of the four chosen deliberately +beats meeting the collision later. + +- **Isolate.** Each agent gets its own instance, ports, schema, or account. Fits + when duplicating the resource is cheap next to the work it unblocks. +- **Serialise the stateful step.** Agents run in parallel, and the dispatcher + lets one at a time through the step that touches the resource. Fits when that + step is short next to the work around it. A built-in lock is not this: it + covers only the thing it guards, leaving whatever that thing mutates still + shared, so the serialised region has to span the change and the verification + that depends on it. +- **Parallel, then verify serially.** Take every agent's result as advisory and + re-run the verification yourself, one at a time. Fits when verification is + cheap, and it is the only option that still holds when an agent ignores its + instructions, so prefer it when unsure. +- **Split the work.** One agent owns everything touching the resource; the rest + parallelise around it. Fits when only some tasks need the resource at all. + +**3. Dispatch with the reporting requirement.** Require each agent to report how +it verified its work, not just the outcome: the command it ran, and whether +every service, device, environment, and account that command touched was its +alone. Say up front that a result which cannot answer that is advisory. A green +result that does not say how it was produced is unverified. + +## The anti-pattern + +An agent reports that it could not obtain its own instance of a shared resource, +used the one already running, and finished green. That is a corrupted run +presented as a workaround. It invalidates its own result rather than excusing +it, and casts the same doubt over every other agent that was using the resource +at the time. diff --git a/skills/parallel-agent-isolation/evals/check_wiring.py b/skills/parallel-agent-isolation/evals/check_wiring.py index a4686e6..706d790 100644 --- a/skills/parallel-agent-isolation/evals/check_wiring.py +++ b/skills/parallel-agent-isolation/evals/check_wiring.py @@ -82,10 +82,10 @@ def check_skill_md() -> None: ) body = path.read_text().split("---", 2)[-1] - # The detection signal is the part of the skill that is easiest to lose in an - # edit and the hardest to notice missing, since the failure it names is silent. - for phrase in ("port is already allocated", "already in use"): - check(phrase in body, f"the body no longer names the collision signal {phrase!r}") + check( + bool(body.strip()), + "SKILL.md has frontmatter but no body, so the skill loads and says nothing", + ) def check_marketplace_entry() -> None: