diff --git a/.gitignore b/.gitignore index 79222ac..51da9a5 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,7 @@ tests/.run-ledger/ # Machine-local operational notes (private paths, spend, secrets workflow) .planning/local/ + +# Python bytecode caches +__pycache__/ +*.pyc diff --git a/.planning/notes/2026-08-31-code-benchmark-battery-plan.md b/.planning/notes/2026-08-31-code-benchmark-battery-plan.md new file mode 100644 index 0000000..1e1ddb9 --- /dev/null +++ b/.planning/notes/2026-08-31-code-benchmark-battery-plan.md @@ -0,0 +1,67 @@ +# Code Benchmark Battery — Implementation Plan + +Date: 2026-08-31 + +## Goal + +Measure whether Co-Evolution workflows produce repository patches that pass +standard deterministic evaluators, while keeping subscription usage bounded and +making every live dispatch auditable before it runs. + +## Conditions + +- **A — Fable solo:** one Fable coding-agent dispatch. +- **B — cross-vendor bounce:** Fable implements; Codex reviews and repairs. +- **C — Fable-led panel:** Fable implements; Codex, GLM, and Kimi critique; + Fable performs the final repair. +- **D — Fable self-bounce:** Fable implements and then reviews/repairs its own + patch. This controls for extra passes and compute. + +Declared provider dispatches are a lower bound: one coding-agent dispatch may +contain multiple internal model turns. The live runner must enforce a declared +Claude-dispatch cap and must never infer a percentage of a Max subscription from +tokens because Anthropic does not publish a fixed weekly token denominator. + +## Delivery sequence + +1. Add frozen suite, condition, external-source, and subset manifests. +2. Add an offline compute estimator and fail-closed prediction validator. +3. Add a pinned SWE-bench installer, metadata fetcher, and official evaluator + bridge under an ignored cache/results tree. +4. Add hermetic tests and wire them into the repository aggregate gate. +5. Prepare the five-task canary and validate official gold scoring when Docker + is available. +6. Add live patch-generation drivers only after the offline harness is green. + Begin with one task and conditions A/B/C, capped at four declared Fable + dispatches. Measure the actual Settings > Usage change before expanding. + +## Safety and scope + +- No full SWE-bench run in this phase. +- No live model calls during setup or hermetic verification. +- No gold patches, hidden tests, or oracle solutions are exposed to agents. +- Every condition starts from the same clean instance and produces a standard + prediction record for the official evaluator. +- Raw datasets, images, virtual environments, trajectories, and results stay + below `benchmarks/results/code/` and remain uncommitted. + +## Acceptance criteria + +- `bash benchmarks/code/code-bench.sh check` passes. +- The estimator reports exact declared dispatch counts and refuses a cap breach + with exit 75. +- Prediction validation rejects unknown instances, duplicates, empty patches, + and malformed JSONL. +- The pinned metadata fetch contains only public task inputs, never gold data. +- The official SWE-bench gold canary passes once Docker is running. +- The repository aggregate test gate includes the new hermetic suite. + +## Execution result + +- Pinned SWE-bench source and CLI installed under the ignored results cache. +- Windows compatibility patch forces LF for the Linux `eval.sh` file only. +- Docker Desktop 29.6.2 official gold canary `sympy__sympy-20916` completed + and resolved 1/1 with zero infrastructure, ambiguous, or evaluator errors. +- Hermetic code-benchmark suite passed 14/14; repository aggregate passed + 42/42 after isolating the intentionally installed private Kimi key. +- No live Fable, Codex, GLM, or Kimi benchmark calls were made during setup. diff --git a/benchmarks/COMPLETE-SUITE-PLAN.md b/benchmarks/COMPLETE-SUITE-PLAN.md new file mode 100644 index 0000000..1f14e1d --- /dev/null +++ b/benchmarks/COMPLETE-SUITE-PLAN.md @@ -0,0 +1,134 @@ +# Complete Testing Suite — Fix & Finish Plan + +Drafted 2026-09-01 (Fable seat), amended same day for the standardized-only +measurement policy. Execute from this file in a fresh Opus session. + +## Measurement policy (Alan, 2026-09-01 — supersedes prior scope) + +All pipeline comparison and ALL shared or published reporting uses +standardized, publicly recognized benchmarks scored by their official +evaluators — currently SWE-bench Verified on the pinned official harness. +Homegrown corpora and judge panels (the bounce-protocol document benchmark, +its 3-judge protocol, blind-judge calibration) are RETIRED from measurement +and from every shared surface. Existing internal results are archived in +place and never published. Rationale: results on a benchmark nobody outside +this repo has seen are not comparable and not worth sharing; common tests +with common baselines are. + +Boundary: the hermetic regression suite (`tests/run-all.sh`, 42 suites) is +engineering QA that gates harness correctness — it is not a benchmark, its +results are not comparison data, and it stays. + +| Surface | Status under policy | +|---|---| +| S1 Code — SWE-bench Verified battery (`benchmarks/code/`) | The measurement surface. Partial: B 5/5 (repair arm inert), C 4/5. A, D, solos unrun | +| S2 Documents — bounce-protocol suite (`benchmarks/`) | RETIRED. Batch b1 complete on disk; archive as internal evidence, no further spend, never on the shared site | +| S3 Regression — `tests/run-all.sh` | QA gate, 42/42 green. Not reported as benchmark data | + +## Issues ledger + +| # | Issue | Status | +|---|---|---| +| 1 | GLM/Kimi bill reasoning against `max_tokens`; capped critics returned empty content | FIXED `735722f` | +| 2 | kimi-seat test could not simulate a missing key with a real key on disk | FIXED `5cf451c` | +| 3 | Codex refuses all writes on Windows despite `--sandbox workspace-write` | OPEN — Phase 0.1 | +| 4 | Conditions A and D never run on code; solos never run | OPEN — Phase 1 | +| 5 | B scored with inert repair arm: 5/5 is really Fable-solo | OPEN — re-run after 0.1 | +| 6 | GLM/Kimi have no agent loop — solo cells need a single-shot harness | OPEN — Phase 1.5 | +| 7 | Judge `position_biased` verdicts discard t1/t7 cells (doc suite) | CLOSED-RETIRED — surface withdrawn; no re-judging spend | +| 8 | `sanitize-leak` on t2 (doc suite) | CLOSED-RETIRED — same | +| 9 | Codex-judge self-preference confound (doc suite) | CLOSED-RETIRED — same | +| 10 | Two orchestrators wrote one status file (b1 watchdog stamped the SWE status) | OPEN — Phase 0.2 | +| 11 | 5-task subset: one task = 20 points; B/C gap is one task | OPEN — Phase 4 decides scale | +| 12 | HF Hub unauthenticated-rate-limit warnings during evaluation | OPEN — minor, Phase 0.3 | +| 13 | Evaluator leaves 5 images per run | OPEN — hygiene, Phase 0.3, default OFF; never delete other projects' images | + +## Phase 0 — Unblock the harness + +**0.1 Codex writable workspace (the critical fix).** +Codex 0.144.5 on Windows degrades `workspace-write` to read-only. Fix +sequence, stop at the first that passes: +1. Probe `-s danger-full-access` with the existing 1-file throwaway-repo test. +2. If refused, probe `--dangerously-bypass-approvals-and-sandbox`. +3. If neither, route codex through WSL against the same workspace path. + +Guardrails: elevated access is acceptable ONLY because benchmark workspaces +are disposable clones under `benchmarks/results/code/runs/`. Gate behind +`CODE_BENCH_CODEX_SANDBOX` (default stays `workspace-write`); record the mode +in `run-manifest.json` — treatment-relevant fact. +Exit: driver-path probe edits a file; mode recorded in manifest. + +**0.2 Status-file single-writer.** One writer per status file; observers get +their own files; every status line carries `writer=`. +Exit: tagged lines present; no cross-suite writes. + +**0.3 Small hygiene.** `HF_TOKEN` via the `.env.local` loader (never echo). +Image-prune stays default OFF. + +## Phase 1 — Complete the code matrix (SWE-bench Verified, frozen 5-task subset) + +Order preserves pairing: never spend Fable dispatches on a condition whose +comparator cannot run. + +| Cell set | Dispatches | Est. cost | Precondition | +|---|---|---|---| +| 1.1 A (Fable solo), 5 cells | 5 Fable | ~$5 | none | +| 1.2 D (self-bounce), 5 cells | 10 Fable | ~$15-20 | none | +| 1.3 B re-run (real repair), 5 cells | 5 Fable + 5 Codex | ~$5 + plan compute | 0.1 | +| 1.4 Codex solo, 5 cells | 5 Codex | plan compute | 0.1 | +| 1.5 GLM solo + Kimi solo, single-shot tier | 10 API calls | cents | new harness | + +1.5 harness: issue text + `git grep`-selected file context in one prompt → +unified diff → `git apply --check` gate → prediction. Label the tier +"single-shot" everywhere — never unlabeled beside agentic rows. +All cells scored by the official Docker evaluator; caps A=1, D=2 on +`--max-claude-dispatches`. +Exit: every matrix row measured or explicitly blocked; zero infrastructure +failures; prediction files validate 5/5 unique frozen IDs. + +## Phase 2 — Retire the homegrown document benchmark + +No model spend. Archive-only: +1. Leave batch b1 results and `reports/b1.md` in place as internal evidence; + they are never published, linked, or summarized on any shared surface. +2. Add a retirement note to `benchmarks/README.md` (doc-suite root): retired + from measurement 2026-09-01 per standardized-only policy; direct readers + to `benchmarks/code/` for the active benchmark. +3. Cancel outstanding doc-suite work: t1/t2/t7 re-judging, sanitizer fix for + judging, blind-judge calibration baselines. Do not delete any code or + results — retire, don't destroy. +Exit: retirement note committed; no doc-suite job scheduled anywhere. + +## Phase 3 — Results site: standardized benchmarks only + +Update the existing artifact (same URL). Two sections: +1. **Leaderboard** — SWE-bench Verified frozen-subset matrix, all Phase 1 + rows, coverage labels, per-task dots. +2. **Methodology & integrity** — evaluator pin + gold canary 1/1, dispatch + counts, per-condition cost, harness commit, and the standing caveat: + frozen 5-task probe, not comparable to published full-500 scores. +Remove nothing that is already standardized; add no homegrown-benchmark +content. One aggregator script (`benchmarks/site/aggregate.sh`) builds a +single JSON from evaluator reports + run logs; the page renders only that. +Exit: site rebuilt from aggregator output alone; every number traceable to a +file on disk; zero references to the retired suite. + +## Phase 4 — Scale gate (go/no-go recommendation, never autonomous) + +Present with dollar estimates, run nothing: +- Expand the SWE-bench Verified subset (25-50 tasks) if any pipeline-vs-solo + gap from Phase 1 is worth confirming. +- Candidate additional suites — standardized public benchmarks only, each + with an official pinned harness (e.g. SWE-bench Lite, Terminal-Bench, + Aider Polyglot, LiveCodeBench). No internal corpus is ever proposed. +- Note for the document pipeline: it currently has NO standardized public + benchmark. Until one exists and is adopted at this gate, document-pipeline + quality claims stay unmeasured rather than internally measured. + +## Budget & sequencing + +Phase 0 is hours, two codex probes. Phase 1 ≈ 20 Fable dispatches (~$25-30), +10 codex cells inside the daily guard cap, GLM/Kimi in cents. Phase 2 is a +docs commit. Phase 3 after Phase 1. Phase 4 is a decision. Throughout: +`.env.local`, results, workspaces, trajectories stay uncommitted; no key +values in logs; one writer per status file; evidence never deleted. diff --git a/benchmarks/README.md b/benchmarks/README.md index 487d3e8..bd3673a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,5 +1,33 @@ # Co-Evolution Benchmark Suite +## Retired from measurement, 2026-09-01 + +The plan-composition benchmark described below no longer measures anything. +Measurement and every shared report now use standardized, publicly recognized +benchmarks scored by their official evaluators — currently SWE-bench Verified +on the pinned official harness in [`benchmarks/code/`](code/README.md). Go +there for the active benchmark. + +A result on a corpus and a judge panel that exist only in this repository +cannot be compared against anything anyone else has run, so it is not worth +publishing. The batch b1 results and `reports/b1.md` stay on disk as internal +evidence of what was built; they are never published, linked, or summarized on +a shared surface. The outstanding work on this suite — re-judging the +position-biased cells, the sanitizer fix for judging, and blind-judge +calibration baselines — is cancelled rather than deferred. + +Nothing here is deleted. The runbook below still describes what the scripts do +if you need to read or re-derive an archived batch. Do not schedule new +batches, and do not add this suite's numbers to any published page. + +A future benchmark may be added only if it is a standardized public suite with +an official pinned evaluator. + +--- + +The rest of this document is the archived runbook for the retired +plan-composition benchmark. + Batch runbook for comparing plan-composition conditions (solo Fable, Codex bounce, panel critique, self-bounce control) on identical planning tasks, scored by three blind automated judges. See `PREREGISTRATION.md` for the diff --git a/benchmarks/code/README.md b/benchmarks/code/README.md new file mode 100644 index 0000000..5036112 --- /dev/null +++ b/benchmarks/code/README.md @@ -0,0 +1,198 @@ +# Code Benchmark Battery + +This suite compares Co-Evolution coding workflows using deterministic external +evaluators. It complements the plan-composition benchmark in `benchmarks/`; +it does not reuse model judges when official tests can decide whether a patch +resolved an issue. + +Authoritative references: [SWE-bench quickstart](https://www.swebench.com/SWE-bench/guides/quickstart/), +[SWE-bench Verified limitations](https://openai.com/index/introducing-swe-bench-verified/), +[Aider benchmark harness](https://github.com/Aider-AI/aider/blob/main/benchmark/README.md), +and [Harbor Terminal-Bench](https://www.harborframework.com/docs/tutorials/running-terminal-bench). + +## Initial battery + +- **SWE-bench Verified canary:** five pinned tasks from five repositories. +- **Local fixture lane:** hermetic contracts for the runner itself; no models. +- **Future lanes:** Aider Polyglot, Terminal-Bench, and a private recent-issue + set can plug into the same prediction and reporting contract. + +The conditions are declared in `conditions.json`, each with a `tier`: + +| Id | Label | Tier | What runs | +|---|---|---|---| +| A | fable-solo | agentic | Fable implements once | +| B | cross-vendor-bounce | agentic | Fable implements; Codex repairs | +| C | fable-led-panel | agentic | Fable implements; Codex/GLM/Kimi critique; Fable repairs | +| D | fable-self-bounce | agentic | Fable implements, then reviews and repairs its own patch | +| E | codex-solo | agentic | Codex implements once | +| F | glm-solo-single-shot | single-shot | GLM sees the issue plus retrieved context, returns one diff | +| G | kimi-solo-single-shot | single-shot | Kimi sees the issue plus retrieved context, returns one diff | + +D is retained as a self-bounce control even when the product question focuses +on A/B/C. E is the comparator that makes B's repair arm interpretable. + +**The tier is not cosmetic.** Agentic conditions run a coding agent with file +tools and test execution. GLM and Kimi are reachable here only as chat +completions, so F and G get one prompt and one answer: no file reads, no test +runs, no second look. A single-shot number is not a like-for-like result +against an agentic one and must never be reported beside one without the +label. + +## Zero-compute setup + +```bash +bash benchmarks/code/code-bench.sh check +bash benchmarks/code/code-bench.sh estimate --suite swebench-verified-canary +bash benchmarks/code/code-bench.sh fetch-metadata +bash benchmarks/code/code-bench.sh setup --install +``` + +These commands make no model calls. The installer pins both the official +SWE-bench repository and dataset revisions from `external-sources.lock.json`. +External files live under the ignored `benchmarks/results/code/` tree. +On Windows, setup also applies the tracked LF-only compatibility patch under +`patches/`; it changes only how the harness writes the Linux `eval.sh` file. + +## Capped patch generation + +Prepare a clean workspace, then run a single condition. Preparation makes no +model calls. `run-workflow` refuses to start without an explicit Claude cap. + +```bash +input=$(bash benchmarks/code/code-bench.sh prepare-instance \ + sympy__sympy-20916 calibration-1 A) + +bash benchmarks/code/code-bench.sh run-workflow \ + --input "$input" \ + --predictions benchmarks/results/code/predictions/calibration-1.jsonl \ + --max-claude-dispatches 1 +``` + +Use `--dry-run` on `run-workflow` to inspect its phase plan without invoking a +provider. Condition C labels its three critiques anonymously and gives Fable +the final repair decision. If `ANTHROPIC_API_KEY` is present, live generation +fails closed so Claude Console credits cannot be charged accidentally instead +of the Max subscription. + +### Codex sandbox mode + +Codex 0.144.5 on Windows accepts `--sandbox workspace-write` and then reports +`sandbox: read-only`, so every write is refused and a repair arm silently goes +inert while still producing a plausible review. `CODE_BENCH_CODEX_SANDBOX` +selects the mode; it defaults to `workspace-write` and the driver records the +value it used in each cell's `run-manifest.json`, because the mode changes what +the treatment actually is. + +```bash +CODE_BENCH_CODEX_SANDBOX=danger-full-access bash benchmarks/code/code-bench.sh run-workflow ... +``` + +Elevated access is defensible only because a benchmark workspace is a +throwaway clone under the ignored `benchmarks/results/code/runs/` tree. Do not +set it for anything else. + +### Single-shot tier + +```bash +bash benchmarks/code/code-bench.sh run-single-shot \ + --input "$input" \ + --predictions benchmarks/results/code/predictions/matrix/F.jsonl \ + --agent glm +``` + +`scripts/select-context.py` picks the files the prompt shows, deterministically +and from public inputs only: paths named in the issue rank first, then files +matched by the issue's rarest identifiers, with tests and examples at half +weight. The model returns a unified diff, `scripts/extract-diff.sh` pulls it +out of the prose, and `git apply --check --recount` gates it; a rejected diff is +fed back with the apply error for up to `CODE_BENCH_SINGLE_SHOT_ATTEMPTS` tries +(default 3). `--recount` recomputes the `@@` line counts from the hunk body and +changes no line of the proposed edit. Without it the gate scores the model's +line arithmetic rather than its patch, which is an artefact of asking for a diff +at all: the agentic conditions edit files directly and never write a hunk +header. Measured on the first two GLM cells, the strict gate rejected every +attempt while `--recount` accepted the first. + +A cell that never produces an applicable patch writes `outcome.json` and +contributes no prediction rather than a broken one. + +`CODE_BENCH_SINGLE_SHOT_MAX_TOKENS` defaults to 32000. Both providers bill +reasoning against `max_tokens`, and on this prompt shape GLM spends roughly +19k reasoning tokens before it writes anything, so a smaller budget returns +`finish_reason=length` with empty content every single time. + +Live phases default to medium reasoning and a 900-second timeout. Override with +`CODE_BENCH_CLAUDE_EFFORT`, `CODE_BENCH_CODEX_EFFORT`, and +`CODE_BENCH_PHASE_TIMEOUT`; changing these values creates a different treatment +and must be recorded in the run manifest. +Direct GLM and Kimi critiques run with bounded reasoning: GLM at +`reasoning_effort=low` (`CODE_BENCH_GLM_REASONING_EFFORT`) and Kimi with +thinking off (`CODE_BENCH_KIMI_THINKING`), under a 2500-token output cap +(`CODE_BENCH_CRITIC_MAX_TOKENS`). Both providers bill reasoning tokens against +`max_tokens`, so an unbounded critic can spend the whole budget before writing +any content and return an empty response. Each critic gets +`CODE_BENCH_CRITIC_ATTEMPTS` attempts (default 3) spaced by +`CODE_BENCH_CRITIC_RETRY_DELAY` seconds; an artifact still invalid after the +last attempt fails the cell instead of reaching the final repair. + +`run-workflow --resume` reuses a successful Fable implementation and valid +critic artifacts. Provider-error text is rejected before the final Fable repair, +so a transient GLM/Kimi failure cannot silently degrade the four-model treatment. + +## Official evaluator + +Docker must be running. Validate the environment with a gold patch before +evaluating generated predictions: + +```bash +bash benchmarks/code/code-bench.sh gold-canary +bash benchmarks/code/code-bench.sh validate-predictions predictions.jsonl +bash benchmarks/code/code-bench.sh evaluate predictions.jsonl +``` + +Prediction JSONL uses the official SWE-bench fields: + +```json +{"instance_id":"owner__repo-123","model_name_or_path":"condition-A","model_patch":"diff --git ..."} +``` + +Gold patches and hidden tests are never stored in this repository or supplied +to a generation workflow. + +## Results site + +```bash +bash benchmarks/site/aggregate.sh +``` + +Builds `benchmarks/results/code/site/leaderboard.json` from the evaluator +reports, the evaluator's own per-instance verdicts, and the run manifests. Each +row carries the report file it came from and each task carries the +`report.json` that decided it, so every number on the published page can be +checked against a file on disk. The page renders that JSON and nothing else. + +## Compute contract + +`estimate` reports declared provider dispatches. For the five-task canary: + +- A/B/C: 20 declared Fable dispatches total. +- A/B/C/D: 30 declared Fable dispatches total. +- One task across A/B/C: 4 declared Fable dispatches. + +Those are lower bounds because a coding-agent session may contain multiple +model turns. Live generation must begin with one task, A/B/C, and a cap of four +declared Fable dispatches. Anthropic does not publish a fixed weekly token +allowance, so the first run is also the calibration: record Settings > Usage +before and after, then use that measured delta to decide whether to expand. + +The batch command enforces that aggregate cap before cloning or dispatching: + +```bash +bash benchmarks/code/code-bench.sh run-canary \ + --run-id calibration-1 \ + --conditions A,B,C \ + --task-limit 1 \ + --max-claude-dispatches 4 \ + --dry-run +``` diff --git a/benchmarks/code/code-bench.sh b/benchmarks/code/code-bench.sh new file mode 100644 index 0000000..2370c25 --- /dev/null +++ b/benchmarks/code/code-bench.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/code-bench-lib.sh +source "$SCRIPT_DIR/lib/code-bench-lib.sh" + +COMMAND="${1:-}" +shift || true + +case "$COMMAND" in + check) + code_check_manifests + ;; + estimate) + exec bash "$SCRIPT_DIR/estimate-compute.sh" "$@" + ;; + fetch-metadata) + suite_json=$(code_suite_json "$(code_suite_id)") + subset=$(code_subset_path "$suite_json") + output=$(code_metadata_path) + exec python "$SCRIPT_DIR/scripts/fetch-swebench-metadata.py" \ + --subset "$subset" --lock "$SCRIPT_DIR/external-sources.lock.json" --output "$output" + ;; + draw-subset) + exec python "$SCRIPT_DIR/scripts/draw-subset.py" \ + --lock "$SCRIPT_DIR/external-sources.lock.json" "$@" + ;; + setup) + exec bash "$SCRIPT_DIR/scripts/setup-swebench.sh" "$@" + ;; + prepare-instance) + exec bash "$SCRIPT_DIR/scripts/prepare-swebench-instance.sh" "$@" + ;; + run-workflow) + exec bash "$SCRIPT_DIR/drivers/run-workflow.sh" "$@" + ;; + run-single-shot) + exec bash "$SCRIPT_DIR/drivers/run-single-shot.sh" "$@" + ;; + run-canary) + exec bash "$SCRIPT_DIR/scripts/run-canary.sh" "$@" + ;; + validate-predictions) + exec bash "$SCRIPT_DIR/validate-predictions.sh" "$@" + ;; + gold-canary) + exec bash "$SCRIPT_DIR/scripts/evaluate-swebench.sh" gold "$@" + ;; + evaluate) + exec bash "$SCRIPT_DIR/scripts/evaluate-swebench.sh" predictions "$@" + ;; + -h|--help|help|"") + cat <<'USAGE' +usage: code-bench.sh COMMAND [options] + + check validate checked-in manifests + estimate [options] report declared provider dispatches + fetch-metadata cache public inputs for the active suite + draw-subset [options] draw a reproducible random subset + setup --check|--install inspect or install pinned SWE-bench tooling + prepare-instance ID RUN COND clone a clean public-input workspace + run-workflow [options] generate one capped condition prediction + run-single-shot [options] generate one single-shot tier prediction + run-canary [options] run a batch with one aggregate Claude cap + validate-predictions FILE validate JSONL before official scoring + gold-canary [INSTANCE] verify the official evaluator with a gold patch + evaluate FILE score generated predictions officially + +No command in this phase invokes a model. Live generation drivers are added +only after this offline harness and its compute cap are verified. +USAGE + ;; + *) + code_die "unknown command: $COMMAND"; exit 2 + ;; +esac diff --git a/benchmarks/code/conditions.json b/benchmarks/code/conditions.json new file mode 100644 index 0000000..bb48f73 --- /dev/null +++ b/benchmarks/code/conditions.json @@ -0,0 +1,54 @@ +{ + "schema": "code-bench-conditions/1.0", + "conditions": [ + { + "id": "A", + "label": "fable-solo", + "tier": "agentic", + "description": "Fable investigates, edits, and tests once.", + "dispatches": {"claude": 1, "codex": 0, "glm": 0, "kimi": 0} + }, + { + "id": "B", + "label": "cross-vendor-bounce", + "tier": "agentic", + "description": "Fable implements; Codex reviews and repairs the patch.", + "dispatches": {"claude": 1, "codex": 1, "glm": 0, "kimi": 0} + }, + { + "id": "C", + "label": "fable-led-panel", + "tier": "agentic", + "description": "Fable implements; Codex, GLM, and Kimi critique; Fable repairs.", + "dispatches": {"claude": 2, "codex": 1, "glm": 1, "kimi": 1} + }, + { + "id": "D", + "label": "fable-self-bounce", + "tier": "agentic", + "description": "Fable implements and performs a second review/repair pass.", + "dispatches": {"claude": 2, "codex": 0, "glm": 0, "kimi": 0} + }, + { + "id": "E", + "label": "codex-solo", + "tier": "agentic", + "description": "Codex investigates, edits, and tests once.", + "dispatches": {"claude": 0, "codex": 1, "glm": 0, "kimi": 0} + }, + { + "id": "F", + "label": "glm-solo-single-shot", + "tier": "single-shot", + "description": "GLM receives the issue plus retrieved file context and returns one unified diff. No agent loop, no test execution.", + "dispatches": {"claude": 0, "codex": 0, "glm": 1, "kimi": 0} + }, + { + "id": "G", + "label": "kimi-solo-single-shot", + "tier": "single-shot", + "description": "Kimi receives the issue plus retrieved file context and returns one unified diff. No agent loop, no test execution.", + "dispatches": {"claude": 0, "codex": 0, "glm": 0, "kimi": 1} + } + ] +} diff --git a/benchmarks/code/drivers/run-single-shot.sh b/benchmarks/code/drivers/run-single-shot.sh new file mode 100644 index 0000000..e34b825 --- /dev/null +++ b/benchmarks/code/drivers/run-single-shot.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +# Single-shot patch generation for models that have no coding-agent loop. +# +# GLM and Kimi are reachable here only as chat completions: no tools, no file +# reads, no test execution. Their cells therefore cannot be a like-for-like +# comparison against the agentic conditions, and the tier is labelled +# "single-shot" in the manifest, the prediction record, and every report so a +# reader never sees these rows unqualified beside an agentic row. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../lib/code-bench-lib.sh +source "$CODE_DIR/lib/code-bench-lib.sh" +# shellcheck source=../../../lib/co-evolution.sh +source "$CODE_BENCH_REPO_ROOT/lib/co-evolution.sh" + +INPUT_JSON="" +PREDICTIONS="" +AGENT="" +DRY_RUN=false +# Both providers bill reasoning against max_tokens. Measured on this prompt +# shape, GLM spends ~19k reasoning tokens before writing a patch, so an 8k +# budget returns finish_reason=length with empty content every time. +MAX_TOKENS="${CODE_BENCH_SINGLE_SHOT_MAX_TOKENS:-32000}" +ATTEMPTS="${CODE_BENCH_SINGLE_SHOT_ATTEMPTS:-3}" +RETRY_DELAY="${CODE_BENCH_SINGLE_SHOT_RETRY_DELAY:-15}" +CONTEXT_FILES="${CODE_BENCH_SINGLE_SHOT_CONTEXT_FILES:-6}" +CONTEXT_BYTES="${CODE_BENCH_SINGLE_SHOT_CONTEXT_BYTES:-24000}" +GLM_CRITIC_REASONING="${CODE_BENCH_GLM_REASONING_EFFORT:-low}" +KIMI_CRITIC_THINKING="${CODE_BENCH_KIMI_THINKING:-disabled}" + +while (( $# > 0 )); do + case "$1" in + --input) INPUT_JSON="${2:?--input needs a value}"; shift 2 ;; + --predictions) PREDICTIONS="${2:?--predictions needs a value}"; shift 2 ;; + --agent) AGENT="${2:?--agent needs a value}"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + *) code_die "unknown single-shot option: $1"; exit 2 ;; + esac +done + +[[ -f "$INPUT_JSON" ]] || { code_die "--input must name a prepared input.json"; exit 2; } +[[ -n "$PREDICTIONS" ]] || { code_die "--predictions is required"; exit 2; } +case "$AGENT" in glm|kimi) ;; *) code_die "--agent must be glm or kimi"; exit 2 ;; esac +for numeric in MAX_TOKENS ATTEMPTS CONTEXT_FILES CONTEXT_BYTES; do + [[ "${!numeric}" =~ ^[1-9][0-9]*$ ]] || { code_die "$numeric must be a positive integer"; exit 2; } +done +[[ "$RETRY_DELAY" =~ ^[0-9]+$ ]] || { code_die "RETRY_DELAY must be a non-negative integer"; exit 2; } + +instance=$(jq -r '.instance_id' "$INPUT_JSON" | tr -d '\r') +condition=$(jq -r '.condition' "$INPUT_JSON" | tr -d '\r') +workspace=$(jq -r '.workspace' "$INPUT_JSON" | tr -d '\r') +task_file=$(jq -r '.task_file' "$INPUT_JSON" | tr -d '\r') +[[ -d "$workspace/.git" && -f "$task_file" ]] || { code_die "prepared workspace or task file is missing"; exit 1; } + +condition_json=$(jq -ce --arg id "$condition" '.conditions | map(select(.id == $id)) | if length == 1 then .[0] else empty end' "$CODE_DIR/conditions.json") \ + || { code_die "unknown condition: $condition"; exit 1; } +tier=$(printf '%s' "$condition_json" | jq -r '.tier' | tr -d '\r') +label=$(printf '%s' "$condition_json" | jq -r '.label' | tr -d '\r') +[[ "$tier" == "single-shot" ]] || { code_die "condition $condition is tier $tier; this driver only runs single-shot cells"; exit 1; } +declared=$(printf '%s' "$condition_json" | jq -r --arg a "$AGENT" '.dispatches[$a]' | tr -d '\r') +(( declared == 1 )) || { code_die "condition $condition does not declare a $AGENT dispatch"; exit 1; } + +results_root=$(cd "$CODE_BENCH_RESULTS_ROOT" && pwd -P) +input_dir=$(cd "$(dirname "$INPUT_JSON")" && pwd -P) +input_abs="$input_dir/$(basename "$INPUT_JSON")" +workspace_abs=$(cd "$workspace" && pwd -P) +task_dir=$(cd "$(dirname "$task_file")" && pwd -P) +task_abs="$task_dir/$(basename "$task_file")" +case "$input_abs" in "$results_root"/runs/*/*/*/input.json) ;; *) code_die "input.json is outside the benchmark run sandbox"; exit 1 ;; esac +[[ "$workspace_abs" == "$input_dir/workspace" ]] || { code_die "workspace does not belong to the prepared cell"; exit 1; } +[[ "$task_abs" == "$input_dir/task.md" ]] || { code_die "task file does not belong to the prepared cell"; exit 1; } +mkdir -p "$(dirname "$PREDICTIONS")" +pred_dir=$(cd "$(dirname "$PREDICTIONS")" && pwd -P) +pred_abs="$pred_dir/$(basename "$PREDICTIONS")" +case "$pred_abs" in "$results_root"/predictions/*) ;; *) code_die "predictions path is outside the benchmark prediction sandbox"; exit 1 ;; esac + +if [[ "$DRY_RUN" == true ]]; then + jq -n --arg instance "$instance" --arg condition "$condition" --arg agent "$AGENT" \ + --arg tier "$tier" --arg label "$label" \ + '{instance:$instance,condition:$condition,agent:$agent,tier:$tier,label:$label, + phases:["select-context","single-shot-diff","git-apply-gate"],executed:false}' + exit 0 +fi + +code_load_env_key ZAI_API_KEY +code_load_env_key KIMI_API_KEY +case "$AGENT" in + glm) [[ -n "${ZAI_API_KEY:-}" ]] || { code_die "condition $condition requires ZAI_API_KEY"; exit 1; } ;; + kimi) [[ -n "${KIMI_API_KEY:-}" ]] || { code_die "condition $condition requires KIMI_API_KEY"; exit 1; } ;; +esac + +cell="$input_dir" +logs="$cell/logs" +mkdir -p "$logs" + +git -C "$workspace" diff --quiet || { code_die "workspace is already dirty; prepare a fresh cell"; exit 1; } + +context_list="$cell/context-files.txt" +python "$CODE_DIR/scripts/select-context.py" \ + --workspace "$workspace" --task "$task_file" --max-files "$CONTEXT_FILES" \ + 2> "$logs/select-context.stderr.log" | tr -d '\r' > "$context_list" \ + || { code_die "context selection failed; see $logs/select-context.stderr.log"; exit 1; } +[[ -s "$context_list" ]] || { code_die "context selection returned no files"; exit 1; } + +model_name="" +case "$AGENT" in + glm) model_name="${GLM_MODEL:-glm-5.3-flash}" ;; + kimi) model_name="${KIMI_MODEL:-kimi-k3}" ;; +esac +jq -n --arg instance "$instance" --arg condition "$condition" --arg agent "$AGENT" \ + --arg tier "$tier" --arg label "$label" --arg model "$model_name" \ + --argjson max_tokens "$MAX_TOKENS" --argjson attempts "$ATTEMPTS" \ + --argjson context_files "$CONTEXT_FILES" --argjson context_bytes "$CONTEXT_BYTES" \ + --rawfile context "$context_list" \ + '{schema:"code-bench-single-shot/1.0",instance:$instance,condition:$condition, + tier:$tier,label:$label,agent:$agent,model:$model, + output_max_tokens:$max_tokens,apply_attempts:$attempts, + retrieval:{max_files:$context_files,max_bytes_per_file:$context_bytes, + selected:($context|split("\n")|map(select(length>0)))}}' \ + > "$cell/run-manifest.json" + +write_prompt() { + local out="$1" feedback="$2" + { + printf '%s\n' "You are fixing a bug in a Python repository. You cannot run commands, open files, or execute tests: everything you may use is below." + printf '%s\n' "Return exactly one unified diff and nothing else, inside a single fenced block that opens with three backticks followed by diff." + printf '%s\n' "Rules for the diff: use git-style headers (diff --git a/PATH b/PATH), keep paths relative to the repository root, include @@ hunk headers with at least three lines of unchanged context, and change only what the issue requires." + printf '%s\n' "Do not add or modify tests. Do not reformat unrelated code. Do not invent files that are not shown." + printf '\n## ISSUE\n\n' + cat "$task_file" + printf '\n## REPOSITORY FILES\n\n' + while IFS= read -r rel; do + [[ -n "$rel" ]] || continue + printf '### %s\n\n' "$rel" + printf '%s\n' '```python' + head -c "$CONTEXT_BYTES" "$workspace/$rel" + printf '\n%s\n\n' '```' + done < <(tr -d '\r' < "$context_list") + if [[ -n "$feedback" && -s "$feedback" ]]; then + printf '\n## YOUR PREVIOUS ATTEMPT DID NOT APPLY\n\n' + printf '%s\n' "git apply rejected the diff below. Produce a corrected diff whose context lines match the files above exactly." + printf '\n### git apply error\n\n' + head -c 4000 "$feedback" + printf '\n' + fi + } > "$out" +} + +invoke_single_shot() { + local prompt="$1" out="$2" err="$3" + case "$AGENT" in + glm) GLM_MAX_TOKENS="$MAX_TOKENS" GLM_REASONING_EFFORT="$GLM_CRITIC_REASONING" \ + invoke_glm "$prompt" "$out" "$err" false ;; + kimi) KIMI_MAX_TOKENS="$MAX_TOKENS" KIMI_THINKING="$KIMI_CRITIC_THINKING" \ + invoke_kimi "$prompt" "$out" "$err" ;; + esac +} + +feedback="" +attempt=1 +applied=false +while (( attempt <= ATTEMPTS )); do + response="$logs/$AGENT-response-$attempt.md" + stderr_log="$logs/$AGENT-response-$attempt.stderr.log" + candidate="$cell/candidate-$attempt.patch" + apply_log="$logs/git-apply-$attempt.log" + write_prompt "$cell/single-shot-prompt-$attempt.md" "$feedback" + invoke_single_shot "$cell/single-shot-prompt-$attempt.md" "$response" "$stderr_log" + + if ! validate_agent_artifact "$response" "$stderr_log" "$AGENT" >/dev/null 2>&1; then + printf 'RETRY: %s attempt %s returned no usable response\n' "$AGENT" "$attempt" >&2 + printf 'provider returned no usable response\n' > "$apply_log" + elif ! bash "$CODE_DIR/scripts/extract-diff.sh" "$response" "$candidate" 2>/dev/null; then + printf 'RETRY: %s attempt %s contained no unified diff\n' "$AGENT" "$attempt" >&2 + printf 'no unified diff found in the response\n' > "$apply_log" + elif git -C "$workspace" apply --check --recount --whitespace=nowarn "$candidate" > "$apply_log" 2>&1; then + git -C "$workspace" apply --recount --whitespace=nowarn "$candidate" >> "$apply_log" 2>&1 \ + || { code_die "git apply --check passed but apply failed; see $apply_log"; exit 1; } + applied=true + break + else + printf 'RETRY: %s attempt %s produced a diff git apply rejected\n' "$AGENT" "$attempt" >&2 + fi + + feedback="$apply_log" + attempt=$((attempt + 1)) + if (( attempt <= ATTEMPTS )); then sleep "$RETRY_DELAY"; fi +done + +if [[ "$applied" != true ]]; then + jq -n --arg instance "$instance" --arg condition "$condition" --arg agent "$AGENT" \ + --arg tier "$tier" --argjson attempts "$ATTEMPTS" \ + '{schema:"code-bench-single-shot-outcome/1.0",instance:$instance,condition:$condition, + agent:$agent,tier:$tier,outcome:"no-applicable-patch",attempts:$attempts}' \ + > "$cell/outcome.json" + code_die "$AGENT produced no applicable patch for $instance after $ATTEMPTS attempts" + exit 1 +fi + +patch="$cell/final.patch" +git -C "$workspace" diff --binary > "$patch" +[[ -s "$patch" ]] || { code_die "single-shot run produced an empty patch"; exit 1; } +record="$cell/prediction.json" +jq -n --arg instance_id "$instance" --arg model "co-evolution-condition-$condition" \ + --rawfile model_patch "$patch" \ + '{instance_id:$instance_id,model_name_or_path:$model,model_patch:$model_patch}' > "$record" +jq -c . "$record" >> "$PREDICTIONS" +jq -n --arg instance "$instance" --arg condition "$condition" --arg agent "$AGENT" \ + --arg tier "$tier" --argjson attempts "$attempt" \ + '{schema:"code-bench-single-shot-outcome/1.0",instance:$instance,condition:$condition, + agent:$agent,tier:$tier,outcome:"patch-applied",attempts:$attempts}' \ + > "$cell/outcome.json" +printf 'WROTE: %s\n' "$record" diff --git a/benchmarks/code/drivers/run-workflow.sh b/benchmarks/code/drivers/run-workflow.sh new file mode 100644 index 0000000..91b2bb2 --- /dev/null +++ b/benchmarks/code/drivers/run-workflow.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../lib/code-bench-lib.sh +source "$CODE_DIR/lib/code-bench-lib.sh" +# shellcheck source=../../../lib/co-evolution.sh +source "$CODE_BENCH_REPO_ROOT/lib/co-evolution.sh" + +INPUT_JSON="" +PREDICTIONS="" +MAX_CLAUDE="" +DRY_RUN=false +RESUME=false +CLAUDE_MODEL="${CODE_BENCH_CLAUDE_MODEL:-fable}" +CODEX_MODEL_LOCAL="${CODE_BENCH_CODEX_MODEL:-gpt-5.6-sol}" +CLAUDE_EFFORT_LOCAL="${CODE_BENCH_CLAUDE_EFFORT:-medium}" +CODEX_EFFORT_LOCAL="${CODE_BENCH_CODEX_EFFORT:-medium}" +PHASE_TIMEOUT="${CODE_BENCH_PHASE_TIMEOUT:-900}" +CRITIC_MAX_TOKENS="${CODE_BENCH_CRITIC_MAX_TOKENS:-2500}" +GLM_CRITIC_REASONING="${CODE_BENCH_GLM_REASONING_EFFORT:-low}" +KIMI_CRITIC_THINKING="${CODE_BENCH_KIMI_THINKING:-disabled}" +CRITIC_ATTEMPTS="${CODE_BENCH_CRITIC_ATTEMPTS:-3}" +CRITIC_RETRY_DELAY="${CODE_BENCH_CRITIC_RETRY_DELAY:-15}" +CODEX_SANDBOX=$(code_codex_sandbox) || exit 2 + +while (( $# > 0 )); do + case "$1" in + --input) INPUT_JSON="${2:?--input needs a value}"; shift 2 ;; + --predictions) PREDICTIONS="${2:?--predictions needs a value}"; shift 2 ;; + --max-claude-dispatches) MAX_CLAUDE="${2:?--max-claude-dispatches needs a value}"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + --resume) RESUME=true; shift ;; + *) code_die "unknown workflow option: $1"; exit 2 ;; + esac +done + +[[ -f "$INPUT_JSON" ]] || { code_die "--input must name a prepared input.json"; exit 2; } +[[ -n "$PREDICTIONS" ]] || { code_die "--predictions is required"; exit 2; } +[[ "$MAX_CLAUDE" =~ ^[0-9]+$ ]] || { code_die "--max-claude-dispatches is required and must be an integer"; exit 2; } +[[ "$PHASE_TIMEOUT" =~ ^[1-9][0-9]*$ ]] || { code_die "CODE_BENCH_PHASE_TIMEOUT must be positive"; exit 2; } +[[ "$CRITIC_MAX_TOKENS" =~ ^[1-9][0-9]*$ ]] || { code_die "CODE_BENCH_CRITIC_MAX_TOKENS must be positive"; exit 2; } +[[ "$CRITIC_ATTEMPTS" =~ ^[1-9][0-9]*$ ]] || { code_die "CODE_BENCH_CRITIC_ATTEMPTS must be positive"; exit 2; } +[[ "$CRITIC_RETRY_DELAY" =~ ^[0-9]+$ ]] || { code_die "CODE_BENCH_CRITIC_RETRY_DELAY must be a non-negative integer"; exit 2; } + +instance=$(jq -r '.instance_id' "$INPUT_JSON" | tr -d '\r') +condition=$(jq -r '.condition' "$INPUT_JSON" | tr -d '\r') +workspace=$(jq -r '.workspace' "$INPUT_JSON" | tr -d '\r') +task_file=$(jq -r '.task_file' "$INPUT_JSON" | tr -d '\r') +[[ -d "$workspace/.git" && -f "$task_file" ]] || { code_die "prepared workspace or task file is missing"; exit 1; } + +results_root=$(cd "$CODE_BENCH_RESULTS_ROOT" && pwd -P) +input_dir=$(cd "$(dirname "$INPUT_JSON")" && pwd -P) +input_abs="$input_dir/$(basename "$INPUT_JSON")" +workspace_abs=$(cd "$workspace" && pwd -P) +task_dir=$(cd "$(dirname "$task_file")" && pwd -P) +task_abs="$task_dir/$(basename "$task_file")" +case "$input_abs" in "$results_root"/runs/*/*/*/input.json) ;; *) code_die "input.json is outside the benchmark run sandbox"; exit 1 ;; esac +[[ "$workspace_abs" == "$input_dir/workspace" ]] || { code_die "workspace does not belong to the prepared cell"; exit 1; } +[[ "$task_abs" == "$input_dir/task.md" ]] || { code_die "task file does not belong to the prepared cell"; exit 1; } +mkdir -p "$(dirname "$PREDICTIONS")" +pred_dir=$(cd "$(dirname "$PREDICTIONS")" && pwd -P) +pred_abs="$pred_dir/$(basename "$PREDICTIONS")" +case "$pred_abs" in "$results_root"/predictions/*) ;; *) code_die "predictions path is outside the benchmark prediction sandbox"; exit 1 ;; esac + +condition_json=$(jq -ce --arg id "$condition" '.conditions | map(select(.id == $id)) | if length == 1 then .[0] else empty end' "$CODE_DIR/conditions.json") \ + || { code_die "unknown condition: $condition"; exit 1; } +claude_needed=$(printf '%s' "$condition_json" | jq -r '.dispatches.claude' | tr -d '\r') +if (( claude_needed > MAX_CLAUDE )); then + printf 'REFUSED: condition %s declares %s Claude dispatches; cap is %s.\n' "$condition" "$claude_needed" "$MAX_CLAUDE" >&2 + exit 75 +fi + +case "$condition" in + A) phases="fable-implement" ;; + B) phases="fable-implement,codex-repair" ;; + C) phases="fable-implement,codex-critique,glm-critique,kimi-critique,fable-repair" ;; + D) phases="fable-implement,fable-self-repair" ;; + E) phases="codex-implement" ;; + F|G) code_die "single-shot conditions run through run-single-shot.sh"; exit 2 ;; +esac +if [[ "$DRY_RUN" == true ]]; then + jq -n --arg instance "$instance" --arg condition "$condition" --arg phases "$phases" \ + --argjson claude "$claude_needed" \ + '{instance:$instance,condition:$condition,phases:($phases|split(",")),declared_claude_dispatches:$claude,executed:false}' + exit 0 +fi + +if [[ -n "${ANTHROPIC_API_KEY:-}" ]]; then + code_die "ANTHROPIC_API_KEY is set; refusing because this would bill API credits instead of the Max plan" + exit 1 +fi +command -v claude >/dev/null 2>&1 || { code_die "claude CLI is required"; exit 1; } +command -v codex >/dev/null 2>&1 || { code_die "codex CLI is required"; exit 1; } + +code_load_env_key ZAI_API_KEY +code_load_env_key KIMI_API_KEY + +cell="$input_dir" +logs="$cell/logs" +reviews="$cell/reviews" +mkdir -p "$logs" "$reviews" +jq -n --arg instance "$instance" --arg condition "$condition" \ + --arg claude_model "$CLAUDE_MODEL" --arg claude_effort "$CLAUDE_EFFORT_LOCAL" \ + --arg codex_model "$CODEX_MODEL_LOCAL" --arg codex_effort "$CODEX_EFFORT_LOCAL" \ + --arg codex_sandbox "$CODEX_SANDBOX" \ + --argjson phase_timeout "$PHASE_TIMEOUT" --argjson declared_claude "$claude_needed" \ + '{schema:"code-bench-run/1.0",instance:$instance,condition:$condition, + models:{claude:$claude_model,codex:$codex_model}, + effort:{claude:$claude_effort,codex:$codex_effort}, + sandbox:{codex:$codex_sandbox}, + phase_timeout_seconds:$phase_timeout,declared_claude_dispatches:$declared_claude}' \ + > "$cell/run-manifest.json" + +write_implement_prompt() { + local out="$1" + { + printf '%s\n\n' "You are solving a repository issue in the current working directory." + printf '%s\n' "Inspect the code, implement the smallest correct fix, and run the most relevant tests available." + printf '%s\n' "Do not read outside the repository. Do not commit. Do not modify tests merely to make a failure disappear." + printf '\n## ISSUE\n\n' + cat "$task_file" + } > "$out" +} + +run_fable() { + local phase="$1" prompt="$2" + local -a cmd=(claude -p --model "$CLAUDE_MODEL" --effort "$CLAUDE_EFFORT_LOCAL" + --safe-mode --permission-mode auto --tools "Bash,Read,Edit,Write,Glob,Grep" + --no-session-persistence --output-format json) + command -v timeout >/dev/null 2>&1 && cmd=(timeout --foreground "${PHASE_TIMEOUT}s" "${cmd[@]}") + (cd "$workspace" && "${cmd[@]}" < "$prompt") \ + > "$logs/$phase.json" 2> "$logs/$phase.stderr.log" + jq -e '.type == "result" and .is_error == false and (.result | type == "string")' \ + "$logs/$phase.json" >/dev/null || { code_die "$phase did not produce a successful Claude result"; return 1; } +} + +run_codex_repair() { + local prompt="$1" + local -a cmd=(codex exec -C "$workspace" -m "$CODEX_MODEL_LOCAL" --sandbox "$CODEX_SANDBOX" + --ephemeral --ignore-user-config -c approval_policy="never" + -c model_reasoning_effort="$CODEX_EFFORT_LOCAL" -) + command -v timeout >/dev/null 2>&1 && cmd=(timeout --foreground "${PHASE_TIMEOUT}s" "${cmd[@]}") + "${cmd[@]}" \ + < "$prompt" > "$logs/codex-repair.log" 2> "$logs/codex-repair.stderr.log" +} + +run_codex_implement() { + local prompt="$1" + local -a cmd=(codex exec -C "$workspace" -m "$CODEX_MODEL_LOCAL" --sandbox "$CODEX_SANDBOX" + --ephemeral --ignore-user-config -c approval_policy="never" + -c model_reasoning_effort="$CODEX_EFFORT_LOCAL" -) + command -v timeout >/dev/null 2>&1 && cmd=(timeout --foreground "${PHASE_TIMEOUT}s" "${cmd[@]}") + "${cmd[@]}" \ + < "$prompt" > "$logs/codex-implement.log" 2> "$logs/codex-implement.stderr.log" +} + +# GLM and Kimi both reason by default and bill reasoning against max_tokens, so +# the capped critic seats must bound reasoning too or they return an empty +# content string. Bounded retries absorb a transient provider hang without +# letting a structurally invalid artifact reach the final repair. +run_direct_critic() { + local agent="$1" out="$2" err="$3" attempt=1 + while (( attempt <= CRITIC_ATTEMPTS )); do + rm -f "$out" + case "$agent" in + glm) GLM_MAX_TOKENS="$CRITIC_MAX_TOKENS" GLM_REASONING_EFFORT="$GLM_CRITIC_REASONING" invoke_glm "$cell/critique-prompt.md" "$out" "$err" false ;; + kimi) KIMI_MAX_TOKENS="$CRITIC_MAX_TOKENS" KIMI_THINKING="$KIMI_CRITIC_THINKING" invoke_kimi "$cell/critique-prompt.md" "$out" "$err" ;; + *) code_die "unknown direct critic: $agent"; return 1 ;; + esac + if validate_agent_artifact "$out" "$err" "$agent" >/dev/null 2>&1; then + return 0 + fi + printf 'RETRY: %s critique attempt %s produced an invalid artifact\n' "$agent" "$attempt" >&2 + attempt=$((attempt + 1)) + # Moonshot enforces org concurrency 1 and answers an overlapping call + # instantly, so an immediate retry just collides again. Back off first. + if (( attempt <= CRITIC_ATTEMPTS )); then sleep "$CRITIC_RETRY_DELAY"; fi + done + return 1 +} + +run_codex_critique() { + local prompt="$1" out="$2" + local -a cmd=(codex exec -C "$workspace" -m "$CODEX_MODEL_LOCAL" --sandbox read-only + --ephemeral --ignore-user-config -c approval_policy="never" + -c model_reasoning_effort="$CODEX_EFFORT_LOCAL" -o "$out" -) + command -v timeout >/dev/null 2>&1 && cmd=(timeout --foreground "${PHASE_TIMEOUT}s" "${cmd[@]}") + "${cmd[@]}" < "$prompt" > "$logs/codex-critique.log" 2> "$logs/codex-critique.stderr.log" +} + +write_implement_prompt "$cell/implement-prompt.md" +if [[ "$condition" == E ]]; then + if [[ "$RESUME" == true && -n "$(git -C "$workspace" diff --name-only)" ]]; then + printf 'REUSED: codex-implement\n' + else + run_codex_implement "$cell/implement-prompt.md" + fi +elif [[ "$RESUME" == true ]] \ + && jq -e '.type == "result" and .is_error == false' "$logs/fable-implement.json" >/dev/null 2>&1 \ + && [[ -n "$(git -C "$workspace" diff --name-only)" ]]; then + printf 'REUSED: fable-implement\n' +else + run_fable fable-implement "$cell/implement-prompt.md" +fi + +case "$condition" in + B) + { + printf '%s\n' "Review the current uncommitted implementation for the issue below. Inspect the diff and repository, correct defects, and run relevant tests. Do not commit." + printf '\n## ISSUE\n\n'; cat "$task_file" + } > "$cell/codex-repair-prompt.md" + run_codex_repair "$cell/codex-repair-prompt.md" + ;; + C) + git -C "$workspace" diff --binary > "$cell/candidate.patch" + { + printf '%s\n' "Critique the candidate patch for correctness, regressions, missing cases, and scope. Do not edit files. Return concrete findings only." + printf '\n## ISSUE\n\n'; cat "$task_file" + printf '\n## CANDIDATE PATCH\n\n'; head -c 120000 "$cell/candidate.patch" + } > "$cell/critique-prompt.md" + if [[ "$RESUME" == true && -s "$reviews/reviewer-1.md" ]] \ + && ! output_is_provider_failure "$reviews/reviewer-1.md"; then + printf 'REUSED: codex-critique\n' + else + run_codex_critique "$cell/critique-prompt.md" "$reviews/reviewer-1.md" + fi + [[ -n "${ZAI_API_KEY:-}" ]] || { code_die "condition C requires ZAI_API_KEY"; exit 1; } + [[ -n "${KIMI_API_KEY:-}" ]] || { code_die "condition C requires KIMI_API_KEY"; exit 1; } + if [[ "$RESUME" != true ]] || ! validate_agent_artifact "$reviews/reviewer-2.md" "$logs/glm-critique.stderr.log" glm >/dev/null 2>&1; then + run_direct_critic glm "$reviews/reviewer-2.md" "$logs/glm-critique.stderr.log" || true + else + printf 'REUSED: glm-critique\n' + fi + validate_agent_artifact "$reviews/reviewer-2.md" "$logs/glm-critique.stderr.log" glm >/dev/null \ + || { code_die "GLM critique is not a valid artifact"; exit 1; } + if [[ "$RESUME" != true ]] || ! validate_agent_artifact "$reviews/reviewer-3.md" "$logs/kimi-critique.stderr.log" kimi >/dev/null 2>&1; then + run_direct_critic kimi "$reviews/reviewer-3.md" "$logs/kimi-critique.stderr.log" || true + else + printf 'REUSED: kimi-critique\n' + fi + validate_agent_artifact "$reviews/reviewer-3.md" "$logs/kimi-critique.stderr.log" kimi >/dev/null \ + || { code_die "Kimi critique is not a valid artifact"; exit 1; } + { + printf '%s\n' "Re-open the current implementation and evaluate the three anonymous reviews below. Decide every finding on its merits, repair accepted issues, and run relevant tests. Do not commit." + printf '\n## ISSUE\n\n'; cat "$task_file" + reviewer_number=0 + for review in "$reviews/reviewer-1.md" "$reviews/reviewer-2.md" "$reviews/reviewer-3.md"; do + reviewer_number=$((reviewer_number + 1)) + printf '\n## REVIEWER %s\n\n' "$reviewer_number" + head -c 40000 "$review" + printf '\n' + done + } > "$cell/fable-repair-prompt.md" + run_fable fable-repair "$cell/fable-repair-prompt.md" + ;; + D) + { + printf '%s\n' "Review your current uncommitted implementation for the issue below. Find and repair correctness or regression risks and run relevant tests. Do not commit." + printf '\n## ISSUE\n\n'; cat "$task_file" + } > "$cell/fable-self-repair-prompt.md" + run_fable fable-self-repair "$cell/fable-self-repair-prompt.md" + ;; +esac + +patch="$cell/final.patch" +git -C "$workspace" diff --binary > "$patch" +[[ -s "$patch" ]] || { code_die "workflow produced an empty patch"; exit 1; } +record="$cell/prediction.json" +jq -n --arg instance_id "$instance" --arg model "co-evolution-condition-$condition" \ + --rawfile model_patch "$patch" \ + '{instance_id:$instance_id,model_name_or_path:$model,model_patch:$model_patch}' > "$record" +jq -c . "$record" >> "$PREDICTIONS" +printf 'WROTE: %s\n' "$record" diff --git a/benchmarks/code/estimate-compute.sh b/benchmarks/code/estimate-compute.sh new file mode 100644 index 0000000..bcec8fa --- /dev/null +++ b/benchmarks/code/estimate-compute.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/code-bench-lib.sh +source "$SCRIPT_DIR/lib/code-bench-lib.sh" + +SUITE="swebench-verified-canary" +CONDITIONS="" +MAX_CLAUDE="" +TASK_LIMIT="" +JSON=false + +while (( $# > 0 )); do + case "$1" in + --suite) SUITE="${2:?--suite needs a value}"; shift 2 ;; + --conditions) CONDITIONS="${2:?--conditions needs a value}"; shift 2 ;; + --max-claude-dispatches) MAX_CLAUDE="${2:?--max-claude-dispatches needs a value}"; shift 2 ;; + --task-limit) TASK_LIMIT="${2:?--task-limit needs a value}"; shift 2 ;; + --json) JSON=true; shift ;; + *) code_die "unknown estimate option: $1"; exit 2 ;; + esac +done + +suite_json=$(code_suite_json "$SUITE") || { code_die "unknown suite: $SUITE"; exit 1; } +subset=$(code_subset_path "$suite_json") +tasks=$(jq '.instances | length' "$subset" | tr -d '\r') +if [[ -n "$TASK_LIMIT" ]]; then + [[ "$TASK_LIMIT" =~ ^[1-9][0-9]*$ ]] || { code_die "--task-limit must be a positive integer"; exit 2; } + (( TASK_LIMIT <= tasks )) || { code_die "--task-limit $TASK_LIMIT exceeds suite size $tasks"; exit 2; } + tasks="$TASK_LIMIT" +fi + +if [[ -z "$CONDITIONS" ]]; then + CONDITIONS=$(printf '%s' "$suite_json" | jq -r '.default_conditions | join(",")' | tr -d '\r') +fi + +selected='[]' +old_ifs=$IFS +IFS=',' +for condition in $CONDITIONS; do + row=$(jq -ce --arg id "$condition" '.conditions | map(select(.id == $id)) | if length == 1 then .[0] else empty end' "$SCRIPT_DIR/conditions.json") \ + || { IFS=$old_ifs; code_die "unknown condition: $condition"; exit 1; } + selected=$(jq -c --argjson row "$row" '. + [$row]' <<<"$selected") +done +IFS=$old_ifs + +summary=$(jq -cn --arg suite "$SUITE" --argjson tasks "$tasks" --argjson conditions "$selected" ' + def total($p): ([$conditions[].dispatches[$p]] | add // 0) * $tasks; + { + schema: "code-bench-compute-estimate/1.0", + suite: $suite, + tasks: $tasks, + conditions: [$conditions[].id], + cells: ($tasks * ($conditions | length)), + declared_dispatches: { + claude: total("claude"), codex: total("codex"), + glm: total("glm"), kimi: total("kimi") + }, + caveat: "Declared dispatches are a lower bound; a coding-agent dispatch may contain multiple model turns." + }') + +claude_calls=$(printf '%s' "$summary" | jq -r '.declared_dispatches.claude' | tr -d '\r') +if [[ -n "$MAX_CLAUDE" ]]; then + [[ "$MAX_CLAUDE" =~ ^[0-9]+$ ]] || { code_die "--max-claude-dispatches must be a non-negative integer"; exit 2; } + if (( claude_calls > MAX_CLAUDE )); then + printf 'REFUSED: estimate requires %s declared Claude dispatches; cap is %s.\n' "$claude_calls" "$MAX_CLAUDE" >&2 + exit 75 + fi +fi + +if [[ "$JSON" == true ]]; then + printf '%s\n' "$summary" | jq . +else + printf 'Compute estimate — %s\n' "$SUITE" + printf ' tasks: %s | conditions: %s | cells: %s\n' \ + "$tasks" "$(printf '%s' "$summary" | jq -r '.conditions | join(",")')" "$(printf '%s' "$summary" | jq -r '.cells')" + for provider in claude codex glm kimi; do + printf ' %-6s declared dispatches: %s\n' "$provider" "$(printf '%s' "$summary" | jq -r --arg p "$provider" '.declared_dispatches[$p]')" + done + printf ' NOTE: one coding-agent dispatch may contain multiple internal model turns.\n' + printf ' Weekly-Max percentage: not derivable without the account usage meter; calibrate with one capped task.\n' +fi diff --git a/benchmarks/code/external-sources.lock.json b/benchmarks/code/external-sources.lock.json new file mode 100644 index 0000000..aaa0a5d --- /dev/null +++ b/benchmarks/code/external-sources.lock.json @@ -0,0 +1,13 @@ +{ + "schema": "code-bench-external-lock/1.0", + "swebench": { + "repository": "https://github.com/SWE-bench/SWE-bench.git", + "commit": "7a21e05772954cc81471ae19d56f436cecf43c54" + }, + "dataset": { + "id": "princeton-nlp/SWE-bench_Verified", + "revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a", + "split": "test" + }, + "compatibility_patches": ["patches/swebench-windows-lf.patch"] +} diff --git a/benchmarks/code/lib/code-bench-lib.sh b/benchmarks/code/lib/code-bench-lib.sh new file mode 100644 index 0000000..c3f89cc --- /dev/null +++ b/benchmarks/code/lib/code-bench-lib.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash + +CODE_BENCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CODE_BENCH_REPO_ROOT="$(cd "$CODE_BENCH_DIR/../.." && pwd)" +CODE_BENCH_RESULTS_ROOT="${CODE_BENCH_RESULTS_ROOT:-$CODE_BENCH_REPO_ROOT/benchmarks/results/code}" + +code_die() { + printf 'ERROR: %s\n' "$*" >&2 + return 1 +} + +code_require() { + command -v "$1" >/dev/null 2>&1 || code_die "$1 is required" +} + +# Which suite the commands operate on. The canary stays the default so every +# existing invocation keeps working; a larger suite is opt-in per command. +code_suite_id() { + printf '%s' "${CODE_BENCH_SUITE:-swebench-verified-canary}" +} + +code_metadata_path() { + printf '%s/metadata/%s.json' "$CODE_BENCH_RESULTS_ROOT" "$(code_suite_id)" +} + +code_suite_json() { + local suite="$1" + jq -ce --arg id "$suite" '.suites[] | select(.id == $id)' "$CODE_BENCH_DIR/suites.json" +} + +code_subset_path() { + local suite_json="$1" rel + rel=$(printf '%s' "$suite_json" | jq -r '.subset_file' | tr -d '\r') + printf '%s/%s' "$CODE_BENCH_DIR" "$rel" +} + +code_check_manifests() { + local failures=0 suite_json subset count declared + code_require jq || return 1 + + jq -e '.schema == "code-bench-conditions/1.0" and + (.conditions | length > 0) and + ([.conditions[].id] | length == (unique | length)) and + (all(.conditions[]; (.dispatches | keys) == ["claude","codex","glm","kimi"])) and + (all(.conditions[]; (.label | type == "string" and length > 0))) and + (all(.conditions[]; .tier == "agentic" or .tier == "single-shot")) and + ([.conditions[].dispatches[] | type == "number" and . >= 0 and floor == .] | all)' \ + "$CODE_BENCH_DIR/conditions.json" >/dev/null || { + printf 'CHECK FAIL: conditions.json\n' >&2; failures=$((failures + 1)); + } + jq -e '.schema == "code-bench-suites/1.0" and + (.suites | length > 0) and + ([.suites[].id] | length == (unique | length))' \ + "$CODE_BENCH_DIR/suites.json" >/dev/null || { + printf 'CHECK FAIL: suites.json\n' >&2; failures=$((failures + 1)); + } + jq -e '.schema == "code-bench-external-lock/1.0" and + (.swebench.commit | test("^[0-9a-f]{40}$")) and + (.dataset.revision | test("^[0-9a-f]{40}$")) and + (.compatibility_patches | type == "array" and length > 0) and + (all(.compatibility_patches[]; test("^patches/[A-Za-z0-9._-]+[.]patch$")))' \ + "$CODE_BENCH_DIR/external-sources.lock.json" >/dev/null || { + printf 'CHECK FAIL: external-sources.lock.json\n' >&2; failures=$((failures + 1)); + } + while IFS= read -r patch_rel; do + [[ -f "$CODE_BENCH_DIR/$patch_rel" ]] || { + printf 'CHECK FAIL: compatibility patch missing: %s\n' "$patch_rel" >&2 + failures=$((failures + 1)) + } + done < <(jq -r '.compatibility_patches[]' "$CODE_BENCH_DIR/external-sources.lock.json" | tr -d '\r') + + while IFS= read -r suite_json; do + subset=$(code_subset_path "$suite_json") + declared=$(printf '%s' "$suite_json" | jq -r '.task_count' | tr -d '\r') + if [[ ! -f "$subset" ]]; then + printf 'CHECK FAIL: subset missing: %s\n' "$subset" >&2 + failures=$((failures + 1)); continue + fi + if ! jq -e '.schema == "code-bench-subset/1.0" and + (.instances | length > 0) and + ([.instances[].instance_id] | length == (unique | length))' "$subset" >/dev/null; then + printf 'CHECK FAIL: malformed or duplicate subset entries: %s\n' "$subset" >&2 + failures=$((failures + 1)); continue + fi + if printf '%s' "$suite_json" | jq -e '.require_unique_repos == true' >/dev/null 2>&1; then + if ! jq -e '[.instances[].repo] | length == (unique | length)' "$subset" >/dev/null; then + printf 'CHECK FAIL: suite requires one task per repository: %s\n' "$subset" >&2 + failures=$((failures + 1)); continue + fi + fi + count=$(jq '.instances | length' "$subset" | tr -d '\r') + if [[ "$count" != "$declared" ]]; then + printf 'CHECK FAIL: suite declares %s tasks but subset contains %s: %s\n' "$declared" "$count" "$subset" >&2 + failures=$((failures + 1)) + fi + while IFS= read -r condition; do + jq -e --arg id "$condition" 'any(.conditions[]; .id == $id)' \ + "$CODE_BENCH_DIR/conditions.json" >/dev/null || { + printf 'CHECK FAIL: suite references unknown condition %s\n' "$condition" >&2 + failures=$((failures + 1)) + } + done < <(printf '%s' "$suite_json" | jq -r '.default_conditions[]' | tr -d '\r') + done < <(jq -c '.suites[]' "$CODE_BENCH_DIR/suites.json") + + (( failures == 0 )) || return 1 + printf 'CHECK: code benchmark manifests PASS\n' +} + +# Codex 0.144.5 on Windows degrades `--sandbox workspace-write` to read-only, +# so the write phases refuse every edit and a repair arm silently goes inert. +# The mode is a treatment-relevant fact, so it is a gated variable rather than +# a constant: elevated access is acceptable only because benchmark workspaces +# are disposable clones under the ignored results tree. +code_codex_sandbox() { + local mode="${CODE_BENCH_CODEX_SANDBOX:-workspace-write}" + case "$mode" in + read-only|workspace-write|danger-full-access) ;; + *) code_die "CODE_BENCH_CODEX_SANDBOX must be read-only, workspace-write, or danger-full-access"; return 1 ;; + esac + printf '%s' "$mode" +} + +# Reads one key from the seat env file without ever echoing its value. Honours +# CO_EVOLVE_ENV_FILE so a test can point at a fixture instead of the real file. +code_load_env_key() { + local name="$1" env_file line value + env_file="${CO_EVOLVE_ENV_FILE:-$CODE_BENCH_REPO_ROOT/.env.local}" + [[ -z "${!name:-}" && -r "$env_file" ]] || return 0 + line=$(grep -m 1 -E "^[[:space:]]*(export[[:space:]]+)?${name}[[:space:]]*=" "$env_file" 2>/dev/null || true) + [[ -n "$line" ]] || return 0 + value=$(printf '%s' "$line" | sed -e 's/^[^=]*=//' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + case "$value" in + \"*\") value="${value#\"}"; value="${value%\"}" ;; + \'*\') value="${value#\'}"; value="${value%\'}" ;; + esac + [[ -n "$value" ]] && printf -v "$name" '%s' "$value" +} + +# Two orchestrators once appended to the same status file and the resulting +# timeline described neither run. A status file now belongs to exactly one +# writer: the owner is recorded on creation, every line carries writer=, and a +# second writer is refused instead of interleaving. +code_status_init() { + local file="$1" writer="$2" + [[ -n "$file" && -n "$writer" ]] || { code_die "code_status_init needs FILE WRITER"; return 1; } + [[ "$writer" =~ ^[A-Za-z0-9._-]+$ ]] || { code_die "writer id must be filesystem-safe: $writer"; return 1; } + mkdir -p "$(dirname "$file")" + if [[ -f "$file.writer" ]] && [[ "$(cat "$file.writer")" != "$writer" ]]; then + code_die "status file $file already belongs to writer $(cat "$file.writer"); use your own file" + return 1 + fi + printf '%s' "$writer" > "$file.writer" + printf 'writer=%s state=running started=%s\n' "$writer" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$file" +} + +code_status_append() { + local file="$1" writer="$2"; shift 2 + [[ -f "$file.writer" ]] || { code_die "status file $file has no owner; call code_status_init first"; return 1; } + [[ "$(cat "$file.writer")" == "$writer" ]] \ + || { code_die "writer $writer may not append to $file (owner $(cat "$file.writer"))"; return 1; } + printf 'writer=%s at=%s %s\n' "$writer" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >> "$file" +} diff --git a/benchmarks/code/patches/swebench-windows-lf.patch b/benchmarks/code/patches/swebench-windows-lf.patch new file mode 100644 index 0000000..6700af5 --- /dev/null +++ b/benchmarks/code/patches/swebench-windows-lf.patch @@ -0,0 +1,14 @@ +diff --git a/swebench/harness/run_evaluation.py b/swebench/harness/run_evaluation.py +--- a/swebench/harness/run_evaluation.py ++++ b/swebench/harness/run_evaluation.py +@@ -359,5 +359,9 @@ def run_instance( + eval_file = Path(log_dir / "eval.sh") +- eval_file.write_text(_inject_asset_restore(test_spec.eval_script, restore_cmds)) ++ eval_file.write_text( ++ _inject_asset_restore(test_spec.eval_script, restore_cmds), ++ encoding="utf-8", ++ newline="\n", ++ ) + logger.info( + f"Eval script for {instance_id} written to {eval_file}; copying to container..." + ) diff --git a/benchmarks/code/schemas/prediction.schema.json b/benchmarks/code/schemas/prediction.schema.json new file mode 100644 index 0000000..89cd943 --- /dev/null +++ b/benchmarks/code/schemas/prediction.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "SWE-bench prediction record", + "type": "object", + "additionalProperties": true, + "required": ["instance_id", "model_name_or_path", "model_patch"], + "properties": { + "instance_id": {"type": "string", "minLength": 1}, + "model_name_or_path": {"type": "string", "minLength": 1}, + "model_patch": {"type": "string", "minLength": 1} + } +} diff --git a/benchmarks/code/scripts/draw-subset.py b/benchmarks/code/scripts/draw-subset.py new file mode 100644 index 0000000..42568f2 --- /dev/null +++ b/benchmarks/code/scripts/draw-subset.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Draw a reproducible random subset of a SWE-bench split. + +The existing canary subset was hand-pinned, one instance per repository, which +makes it useful for harness development and useless for estimating a score: a +non-random subset cannot be extrapolated to the full split at any sample size. +This draws a uniform random sample instead, from the same revision the lock file +pins, so the result is an unbiased estimate of the full-split score. + +The seed is written into the subset file. Re-running with the same seed and the +same dataset revision reproduces the identical sample. +""" +from __future__ import annotations + +import argparse +import json +import random +import urllib.parse +import urllib.request +from pathlib import Path + + +def get_json(url: str) -> dict: + with urllib.request.urlopen(url, timeout=60) as response: + return json.load(response) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--lock", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--suite-id", required=True) + parser.add_argument("--size", type=int, required=True) + parser.add_argument("--seed", type=int, required=True) + parser.add_argument("--split", default="test") + parser.add_argument("--total", type=int, default=500) + args = parser.parse_args() + + lock = json.loads(args.lock.read_text(encoding="utf-8")) + dataset_id = lock["dataset"]["id"] + expected = lock["dataset"]["revision"] + actual = get_json(f"https://huggingface.co/api/datasets/{dataset_id}").get("sha") + if actual != expected: + raise SystemExit(f"dataset revision drift: expected {expected}, got {actual}") + + api_id = urllib.parse.quote(dataset_id, safe="") + population: list[dict] = [] + for offset in range(0, args.total, 100): + url = ( + "https://datasets-server.huggingface.co/rows" + f"?dataset={api_id}&config=default&split={args.split}" + f"&offset={offset}&length=100" + ) + for wrapped in get_json(url).get("rows", []): + row = wrapped["row"] + population.append({ + "instance_id": row["instance_id"], + "repo": row["repo"], + }) + + if len(population) != args.total: + raise SystemExit(f"expected {args.total} instances, enumerated {len(population)}") + if args.size > len(population): + raise SystemExit("sample size exceeds the split") + + # Sort first so the draw depends only on the seed, never on server row order. + population.sort(key=lambda r: r["instance_id"]) + sample = random.Random(args.seed).sample(population, args.size) + sample.sort(key=lambda r: r["instance_id"]) + + payload = { + "schema": "code-bench-subset/1.0", + "suite_id": args.suite_id, + "sampling": { + "method": "uniform-random-without-replacement", + "seed": args.seed, + "drawn_from": {"dataset": dataset_id, "revision": expected, + "split": args.split, "population": len(population)}, + "note": ("A uniform random draw, so the observed score is an unbiased " + "estimate of the full-split score with a binomial interval."), + }, + "instances": sample, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + repos: dict[str, int] = {} + for row in sample: + repos[row["repo"]] = repos.get(row["repo"], 0) + 1 + print(f"WROTE: {len(sample)} of {len(population)} -> {args.output}") + for repo, count in sorted(repos.items(), key=lambda kv: (-kv[1], kv[0])): + print(f" {count:3d} {repo}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/code/scripts/evaluate-swebench.sh b/benchmarks/code/scripts/evaluate-swebench.sh new file mode 100644 index 0000000..efd29eb --- /dev/null +++ b/benchmarks/code/scripts/evaluate-swebench.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../lib/code-bench-lib.sh +source "$CODE_DIR/lib/code-bench-lib.sh" + +MODE="${1:-}" +shift || true +SUITE=$(code_suite_id) +CACHE="$CODE_BENCH_RESULTS_ROOT/.cache" +if [[ -x "$CACHE/venv/Scripts/swebench.exe" ]]; then CLI="$CACHE/venv/Scripts/swebench.exe"; else CLI="$CACHE/venv/bin/swebench"; fi + +[[ -x "$CLI" ]] || { code_die "SWE-bench is not installed; run code-bench.sh setup --install"; exit 1; } +# Python on Windows otherwise inherits CP1252 for pathlib.write_text(), which +# cannot encode some Unicode symbols present in official evaluation scripts. +export PYTHONUTF8=1 +export PYTHONIOENCODING=utf-8 +export HF_HUB_DISABLE_SYMLINKS_WARNING=1 +# Unauthenticated Hub reads are rate-limited and the warning fires mid-run. A +# token is optional; it is read from the seat env file and never echoed. +code_load_env_key HF_TOKEN +if [[ -n "${HF_TOKEN:-}" ]]; then export HF_TOKEN; fi +if command -v timeout >/dev/null 2>&1; then + timeout 10 docker info >/dev/null 2>&1 || { code_die "Docker engine is not running"; exit 1; } +else + docker info >/dev/null 2>&1 || { code_die "Docker engine is not running"; exit 1; } +fi +EVAL_ROOT="$CODE_BENCH_RESULTS_ROOT/evaluation" +mkdir -p "$EVAL_ROOT" + +case "$MODE" in + gold) + suite_json=$(code_suite_json "$SUITE") + subset=$(code_subset_path "$suite_json") + instance="${1:-$(jq -r '.instances[0].instance_id' "$subset" | tr -d '\r')}" + jq -e --arg id "$instance" '.instances[] | select(.instance_id == $id)' "$subset" >/dev/null \ + || { code_die "gold instance is outside frozen subset: $instance"; exit 1; } + run_id="gold-canary-$(date -u +%Y%m%dT%H%M%SZ)" + (cd "$EVAL_ROOT" && "$CLI" eval verified --gold -i "$instance" --run-id "$run_id" -j 1) + ;; + predictions) + predictions="${1:-}" + [[ -n "$predictions" ]] || { code_die "predictions mode needs a JSONL file"; exit 2; } + bash "$CODE_DIR/validate-predictions.sh" "$predictions" "$SUITE" + predictions_dir=$(cd "$(dirname "$predictions")" && pwd -P) + predictions="$predictions_dir/$(basename "$predictions")" + run_id="code-bench-$(date -u +%Y%m%dT%H%M%SZ)" + (cd "$EVAL_ROOT" && "$CLI" eval verified -p "$predictions" --run-id "$run_id" -j "${CODE_BENCH_EVAL_JOBS:-1}") + ;; + *) + code_die "usage: evaluate-swebench.sh gold [INSTANCE]|predictions FILE"; exit 2 + ;; +esac diff --git a/benchmarks/code/scripts/extract-diff.sh b/benchmarks/code/scripts/extract-diff.sh new file mode 100644 index 0000000..09ca6d2 --- /dev/null +++ b/benchmarks/code/scripts/extract-diff.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Pull the unified diff out of a chat response. +# +# Single-shot seats answer in prose-plus-fence, and the fence label varies +# (```diff, ```patch, or bare ```). Some answers skip the fence and trail +# explanation after the patch. This reads the response and writes just the +# diff; it exits non-zero when the response contains no git-style diff at all, +# which is a cell failure rather than a patch. +set -euo pipefail + +RESPONSE="${1:-}" +OUT="${2:-}" +[[ -f "$RESPONSE" && -n "$OUT" ]] || { printf 'usage: extract-diff.sh RESPONSE OUT\n' >&2; exit 2; } + +raw="${OUT}.raw" +awk ' + /^[[:space:]]*```/ { + if (inblock) { exit } + if (seen == 0) { inblock = 1; seen = 1; next } + next + } + inblock { print } +' "$RESPONSE" > "$raw" +grep -q '^diff --git ' "$raw" || cp "$RESPONSE" "$raw" + +# Keep the run of diff lines and drop the prose on either side of it. Anything +# that is not recognisable diff syntax ends the patch. +awk ' + BEGIN { started = 0 } + !started && /^diff --git / { started = 1 } + started { + if ($0 ~ /^(diff --git |index |--- |\+\+\+ |@@ |[+-]|[[:space:]]|\\ No newline|new file mode |deleted file mode |old mode |new mode |similarity index |rename from |rename to |Binary files |GIT binary patch)/ || $0 == "") { + print + } else { + exit + } + } +' "$raw" > "$OUT" +rm -f "$raw" + +grep -q '^diff --git ' "$OUT" diff --git a/benchmarks/code/scripts/fetch-swebench-metadata.py b/benchmarks/code/scripts/fetch-swebench-metadata.py new file mode 100644 index 0000000..f23fd8e --- /dev/null +++ b/benchmarks/code/scripts/fetch-swebench-metadata.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Fetch only public SWE-bench task inputs for the frozen canary subset.""" + +from __future__ import annotations + +import argparse +import json +import urllib.parse +import urllib.request +from pathlib import Path + + +SAFE_FIELDS = ( + "instance_id", + "repo", + "base_commit", + "problem_statement", + "created_at", + "version", +) + + +def get_json(url: str) -> dict: + with urllib.request.urlopen(url, timeout=60) as response: + return json.load(response) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--subset", type=Path, required=True) + parser.add_argument("--lock", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + subset = json.loads(args.subset.read_text(encoding="utf-8")) + lock = json.loads(args.lock.read_text(encoding="utf-8")) + wanted = [item["instance_id"] for item in subset["instances"]] + wanted_set = set(wanted) + + dataset_id = lock["dataset"]["id"] + dataset_info = get_json(f"https://huggingface.co/api/datasets/{dataset_id}") + actual_revision = dataset_info.get("sha") + expected_revision = lock["dataset"]["revision"] + if actual_revision != expected_revision: + raise SystemExit( + f"dataset revision drift: expected {expected_revision}, got {actual_revision}" + ) + + api_id = urllib.parse.quote(dataset_id, safe="") + found: dict[str, dict] = {} + for offset in range(0, 500, 100): + url = ( + "https://datasets-server.huggingface.co/rows" + f"?dataset={api_id}&config=default&split=test&offset={offset}&length=100" + ) + for wrapped in get_json(url).get("rows", []): + row = wrapped["row"] + if row.get("instance_id") in wanted_set: + found[row["instance_id"]] = {key: row.get(key) for key in SAFE_FIELDS} + + missing = [instance for instance in wanted if instance not in found] + if missing: + raise SystemExit(f"missing frozen instances: {', '.join(missing)}") + + payload = { + "schema": "code-bench-public-inputs/1.0", + "dataset": lock["dataset"], + "instances": [found[instance] for instance in wanted], + } + forbidden = {"patch", "test_patch", "FAIL_TO_PASS", "PASS_TO_PASS"} + if any(forbidden.intersection(instance) for instance in payload["instances"]): + raise SystemExit("refusing to write gold or hidden-test fields") + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + print(f"WROTE: {len(wanted)} public task inputs -> {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/code/scripts/prepare-swebench-instance.sh b/benchmarks/code/scripts/prepare-swebench-instance.sh new file mode 100644 index 0000000..197286e --- /dev/null +++ b/benchmarks/code/scripts/prepare-swebench-instance.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../lib/code-bench-lib.sh +source "$CODE_DIR/lib/code-bench-lib.sh" + +INSTANCE="${1:-}" +RUN_ID="${2:-}" +CONDITION="${3:-}" +[[ -n "$INSTANCE" && -n "$RUN_ID" && -n "$CONDITION" ]] || { + code_die "usage: prepare-swebench-instance.sh INSTANCE RUN_ID CONDITION"; exit 2; +} +[[ "$RUN_ID" =~ ^[A-Za-z0-9._-]+$ ]] || { code_die "unsafe run id: $RUN_ID"; exit 2; } +[[ "$CONDITION" =~ ^[A-Za-z0-9_-]+$ ]] || { code_die "unsafe condition id: $CONDITION"; exit 2; } +jq -e --arg id "$CONDITION" 'any(.conditions[]; .id == $id)' "$CODE_DIR/conditions.json" >/dev/null \ + || { code_die "unknown condition: $CONDITION"; exit 2; } + +METADATA=$(code_metadata_path) +[[ -f "$METADATA" ]] || { code_die "public metadata is absent; run code-bench.sh fetch-metadata"; exit 1; } +row=$(jq -ce --arg id "$INSTANCE" '.instances[] | select(.instance_id == $id)' "$METADATA") \ + || { code_die "instance is outside suite $(code_suite_id): $INSTANCE"; exit 1; } +repo=$(printf '%s' "$row" | jq -r '.repo' | tr -d '\r') +base_commit=$(printf '%s' "$row" | jq -r '.base_commit' | tr -d '\r') +[[ "$repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || { code_die "unsafe repository id: $repo"; exit 1; } +[[ "$base_commit" =~ ^[0-9a-f]{40}$ ]] || { code_die "unsafe base commit for $INSTANCE"; exit 1; } + +CELL="$CODE_BENCH_RESULTS_ROOT/runs/$RUN_ID/$INSTANCE/$CONDITION" +WORKSPACE="$CELL/workspace" +TASK_FILE="$CELL/task.md" +if [[ -e "$CELL" ]]; then + code_die "cell already exists; choose a new run id: $CELL"; exit 1 +fi +mkdir -p "$CELL" + +git clone --filter=blob:none --no-checkout "https://github.com/$repo.git" "$WORKSPACE" +git -C "$WORKSPACE" checkout --detach "$base_commit" + +# A clone carries every commit AFTER the base, including the upstream fix for +# this very issue. An agent that runs `git log origin/main` can read the answer +# instead of deriving it, and observed runs did exactly that -- citing the +# upstream PR and commit hash back in their reports. Replace the history with a +# single synthetic commit holding the base tree, so the working tree is +# identical but nothing about the future is recoverable. `git diff` still yields +# the agent's changes, which is all the driver needs. +rm -rf "$WORKSPACE/.git" +git -C "$WORKSPACE" init -q +git -C "$WORKSPACE" -c core.autocrlf=false add -A +git -C "$WORKSPACE" -c user.email=bench@local -c user.name=bench \ + commit -q -m "base $base_commit" --no-gpg-sign +if git -C "$WORKSPACE" log --oneline --all | wc -l | grep -qv '^ *1$'; then + code_die "workspace history was not reduced to a single commit"; exit 1 +fi +if git -C "$WORKSPACE" remote -v | grep -q .; then + code_die "workspace still has a remote configured"; exit 1 +fi +git -C "$WORKSPACE" status --porcelain | grep -q . && { + code_die "prepared workspace is unexpectedly dirty: $WORKSPACE"; exit 1; +} +printf '%s\n' "$row" | jq -r '.problem_statement' > "$TASK_FILE" +jq -n \ + --arg instance_id "$INSTANCE" --arg condition "$CONDITION" \ + --arg repo "$repo" --arg base_commit "$base_commit" \ + --arg workspace "$WORKSPACE" --arg task_file "$TASK_FILE" \ + '{schema:"code-bench-cell-input/1.0", instance_id:$instance_id, + condition:$condition, repo:$repo, base_commit:$base_commit, + workspace:$workspace, task_file:$task_file}' > "$CELL/input.json" +printf '%s\n' "$CELL/input.json" diff --git a/benchmarks/code/scripts/run-canary.sh b/benchmarks/code/scripts/run-canary.sh new file mode 100644 index 0000000..6efba12 --- /dev/null +++ b/benchmarks/code/scripts/run-canary.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../lib/code-bench-lib.sh +source "$CODE_DIR/lib/code-bench-lib.sh" + +RUN_ID="" +CONDITIONS="A,B,C" +TASK_LIMIT=1 +MAX_CLAUDE="" +DRY_RUN=false + +while (( $# > 0 )); do + case "$1" in + --run-id) RUN_ID="${2:?--run-id needs a value}"; shift 2 ;; + --conditions) CONDITIONS="${2:?--conditions needs a value}"; shift 2 ;; + --task-limit) TASK_LIMIT="${2:?--task-limit needs a value}"; shift 2 ;; + --max-claude-dispatches) MAX_CLAUDE="${2:?--max-claude-dispatches needs a value}"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + *) code_die "unknown canary option: $1"; exit 2 ;; + esac +done + +[[ "$RUN_ID" =~ ^[A-Za-z0-9._-]+$ ]] || { code_die "--run-id is required and must be filesystem-safe"; exit 2; } +[[ "$TASK_LIMIT" =~ ^[1-9][0-9]*$ ]] || { code_die "--task-limit must be positive"; exit 2; } +[[ "$MAX_CLAUDE" =~ ^[0-9]+$ ]] || { code_die "--max-claude-dispatches is required"; exit 2; } + +bash "$CODE_DIR/estimate-compute.sh" --suite swebench-verified-canary \ + --conditions "$CONDITIONS" --task-limit "$TASK_LIMIT" \ + --max-claude-dispatches "$MAX_CLAUDE" + +if [[ "$DRY_RUN" == true ]]; then + printf 'DRY RUN: no repositories cloned and no providers invoked.\n' + exit 0 +fi + +suite_json=$(code_suite_json "swebench-verified-canary") +subset=$(code_subset_path "$suite_json") +pred_dir="$CODE_BENCH_RESULTS_ROOT/predictions/$RUN_ID" +mkdir -p "$pred_dir" + +task_index=0 +while IFS= read -r instance; do + task_index=$((task_index + 1)) + (( task_index <= TASK_LIMIT )) || break + old_ifs=$IFS + IFS=',' + for condition in $CONDITIONS; do + input=$(bash "$CODE_DIR/scripts/prepare-swebench-instance.sh" "$instance" "$RUN_ID" "$condition") + per_condition=$(jq -r --arg id "$condition" '.conditions[] | select(.id == $id) | .dispatches.claude' \ + "$CODE_DIR/conditions.json" | tr -d '\r') + bash "$CODE_DIR/drivers/run-workflow.sh" --input "$input" \ + --predictions "$pred_dir/$condition.jsonl" \ + --max-claude-dispatches "$per_condition" + done + IFS=$old_ifs +done < <(jq -r '.instances[].instance_id' "$subset" | tr -d '\r') + +for predictions in "$pred_dir"/*.jsonl; do + bash "$CODE_DIR/validate-predictions.sh" "$predictions" swebench-verified-canary +done +printf 'COMPLETE: canary predictions -> %s\n' "$pred_dir" diff --git a/benchmarks/code/scripts/select-context.py b/benchmarks/code/scripts/select-context.py new file mode 100644 index 0000000..f80ab8e --- /dev/null +++ b/benchmarks/code/scripts/select-context.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Pick the repository files a single-shot patch attempt should see. + +The single-shot tier has no agent loop: the model cannot open files, so the +harness must choose the context. Selection is deterministic and uses only +public inputs (the issue text and the repository at its base commit) so the +same cell always builds the same prompt. +""" +import argparse +import re +import subprocess +import sys + +SOURCE_SUFFIXES = ('.py', '.pyx', '.pyi') + +# Words that appear in nearly every issue report and match nearly every file. +STOPWORDS = frozenset(""" +about above after again against all also always analysis and another any are +around because been before being below between both build call called calls +can case cases change changed changes check code coming compare could current +currently data default depending description details different does doing done +during each either else error errors even every example expect expected +failing fails false first fixed follow following from function functions +generate get gets github given gives group handle has have here how however +implementation import instead into issue itself just keep known last later +like line lines list little look make makes many maybe method methods might +more most much must name names need needs never new none not note now number +object objects only open option options order other output outside over own +part pass patch please point possible previous print problem produce python +raise raised raises rather read really reason regression release report +reported reproduce result results return returns run running same seems self +set should show shown side similar simple since small some something still +such support sure take test tests than that the their them then there these +they thing think this those three through time trying two type types under +until update use used user uses using value values version very want was way +what when where whether which while will with within without work working +would write wrong your +""".split()) + +FILE_PATH_RE = re.compile(r'\b((?:[\w.-]+/)+[\w.-]+\.(?:py|pyx|pyi))\b') +BACKTICK_RE = re.compile(r'`([^`\n]{2,120})`') +IDENT_RE = re.compile(r'\b([A-Za-z_][A-Za-z0-9_]{3,})\b') + + +def git(workspace, *args): + proc = subprocess.run(('git', '-C', workspace) + args, + capture_output=True, text=True, errors='replace') + return proc.returncode, proc.stdout + + +def tracked_source_files(workspace): + rc, out = git(workspace, 'ls-files') + if rc != 0: + sys.exit('ERROR: git ls-files failed in %s' % workspace) + return set(p for p in out.splitlines() if p.endswith(SOURCE_SUFFIXES)) + + +def candidate_tokens(issue, limit): + """Rank issue tokens: backticked and dotted names first, then identifiers.""" + weighted = {} + + def add(token, weight): + token = token.strip() + if len(token) < 4 or token.lower() in STOPWORDS: + return + weighted[token] = max(weighted.get(token, 0.0), weight) + + for span in BACKTICK_RE.findall(issue): + add(span, 3.0) + for part in re.split(r'[^A-Za-z0-9_]+', span): + add(part, 2.0) + for ident in IDENT_RE.findall(issue): + add(ident, 1.0) + + ordered = sorted(weighted.items(), key=lambda kv: (-kv[1], kv[0])) + return [token for token, _ in ordered[:limit]] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--workspace', required=True) + ap.add_argument('--task', required=True) + ap.add_argument('--max-files', type=int, default=6) + ap.add_argument('--max-tokens', type=int, default=40) + args = ap.parse_args() + + # Bash callers read this list line by line; Windows text mode would append + # a carriage return to every path and break the lookups downstream. + sys.stdout.reconfigure(newline=chr(10)) + + with open(args.task, encoding='utf-8', errors='replace') as handle: + issue = handle.read() + + tracked = tracked_source_files(args.workspace) + if not tracked: + sys.exit('ERROR: workspace has no tracked Python sources') + + scores = {} + + # A path spelled out in the issue is the strongest possible signal. + for path in FILE_PATH_RE.findall(issue): + for candidate in tracked: + if candidate == path or candidate.endswith('/' + path): + scores[candidate] = scores.get(candidate, 0.0) + 10.0 + + for token in candidate_tokens(issue, args.max_tokens): + rc, out = git(args.workspace, 'grep', '-l', '-F', '--', token) + if rc != 0: + continue + hits = [p for p in out.splitlines() if p in tracked] + # A token matching half the repository says nothing about where the bug + # is; a token matching three files says a great deal. + if not hits or len(hits) > 40: + continue + share = 1.0 / len(hits) + for path in hits: + scores[path] = scores.get(path, 0.0) + share + + if not scores: + sys.exit('ERROR: no candidate files matched the issue text') + + # Test and example modules are legitimate context but they crowd out the + # source file the patch has to touch, so they compete at half weight. + for path in list(scores): + parts = path.split('/') + is_support = ('tests' in parts or 'test' in parts + or 'examples' in parts or parts[-1].startswith('test_')) + if is_support: + scores[path] *= 0.5 + + ranked = sorted(scores.items(), key=lambda kv: (-kv[1], kv[0])) + for path, _ in ranked[:args.max_files]: + print(path) + + +if __name__ == '__main__': + main() diff --git a/benchmarks/code/scripts/setup-swebench.sh b/benchmarks/code/scripts/setup-swebench.sh new file mode 100644 index 0000000..ed640b2 --- /dev/null +++ b/benchmarks/code/scripts/setup-swebench.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../lib/code-bench-lib.sh +source "$CODE_DIR/lib/code-bench-lib.sh" + +MODE="${1:---check}" +CACHE="$CODE_BENCH_RESULTS_ROOT/.cache" +REPO="$CACHE/SWE-bench" +VENV="$CACHE/venv" +LOCK="$CODE_DIR/external-sources.lock.json" +SHA=$(jq -r '.swebench.commit' "$LOCK" | tr -d '\r') +URL=$(jq -r '.swebench.repository' "$LOCK" | tr -d '\r') + +python_path() { + if [[ -x "$VENV/Scripts/python.exe" ]]; then printf '%s' "$VENV/Scripts/python.exe"; else printf '%s' "$VENV/bin/python"; fi +} + +cli_path() { + if [[ -x "$VENV/Scripts/swebench.exe" ]]; then printf '%s' "$VENV/Scripts/swebench.exe"; else printf '%s' "$VENV/bin/swebench"; fi +} + +check_state() { + local failed=0 + for tool in git uv docker jq; do + if command -v "$tool" >/dev/null 2>&1; then printf '%s=present\n' "$tool"; else printf '%s=missing\n' "$tool"; failed=1; fi + done + if command -v timeout >/dev/null 2>&1; then + timeout 10 docker info >/dev/null 2>&1 && engine_ready=true || engine_ready=false + else + docker info >/dev/null 2>&1 && engine_ready=true || engine_ready=false + fi + if [[ "$engine_ready" == true ]]; then printf 'docker_engine=ready\n'; else printf 'docker_engine=unavailable\n'; failed=1; fi + if [[ -d "$REPO/.git" && "$(git -C "$REPO" rev-parse HEAD 2>/dev/null)" == "$SHA" ]]; then + printf 'swebench_source=pinned\n' + else + printf 'swebench_source=not-installed\n'; failed=1 + fi + patches_ready=true + while IFS= read -r patch_rel; do + patch_file="$CODE_DIR/$patch_rel" + if [[ ! -f "$patch_file" ]] || ! git -C "$REPO" apply --reverse --check "$patch_file" >/dev/null 2>&1; then + patches_ready=false + fi + done < <(jq -r '.compatibility_patches[]' "$LOCK" | tr -d '\r') + if [[ "$patches_ready" == true ]]; then printf 'swebench_patches=ready\n'; else printf 'swebench_patches=missing\n'; failed=1; fi + if [[ -x "$(cli_path)" ]]; then printf 'swebench_cli=ready\n'; else printf 'swebench_cli=not-installed\n'; failed=1; fi + return "$failed" +} + +case "$MODE" in + --check) + check_state + ;; + --install) + code_require git; code_require uv; code_require jq + mkdir -p "$CACHE" + if [[ ! -d "$REPO/.git" ]]; then + git clone --filter=blob:none --no-checkout "$URL" "$REPO" + fi + git -C "$REPO" fetch --depth 1 origin "$SHA" +git -C "$REPO" checkout --detach "$SHA" +while IFS= read -r patch_rel; do + patch_file="$CODE_DIR/$patch_rel" + [[ -f "$patch_file" ]] || { code_die "compatibility patch is missing: $patch_file"; exit 1; } + if git -C "$REPO" apply --reverse --check "$patch_file" >/dev/null 2>&1; then + printf 'PATCHED: %s (already applied)\n' "$patch_rel" + elif git -C "$REPO" apply --check "$patch_file"; then + git -C "$REPO" apply "$patch_file" + printf 'PATCHED: %s\n' "$patch_rel" + else + code_die "compatibility patch does not apply cleanly: $patch_rel"; exit 1 + fi +done < <(jq -r '.compatibility_patches[]' "$LOCK" | tr -d '\r') +if [[ ! -x "$(python_path)" ]]; then + uv venv --python 3.11 "$VENV" +fi +if [[ ! -x "$(cli_path)" ]]; then + uv pip install --python "$(python_path)" -e "$REPO" +else + printf 'REUSED: existing editable SWE-bench installation\n' +fi +printf 'INSTALLED: SWE-bench %s\n' "$SHA" + check_state || true + ;; + *) + code_die "usage: setup-swebench.sh --check|--install"; exit 2 + ;; +esac diff --git a/benchmarks/code/subsets/swebench-verified-canary.json b/benchmarks/code/subsets/swebench-verified-canary.json new file mode 100644 index 0000000..2ba0bdc --- /dev/null +++ b/benchmarks/code/subsets/swebench-verified-canary.json @@ -0,0 +1,12 @@ +{ + "schema": "code-bench-subset/1.0", + "id": "swebench-verified-canary", + "selection": "Choose the shortest public problem statement within each repository, then take the five shortest across repositories in the pinned 500-task dataset; frozen before any live run.", + "instances": [ + {"instance_id": "sympy__sympy-20916", "repo": "sympy/sympy"}, + {"instance_id": "django__django-16819", "repo": "django/django"}, + {"instance_id": "scikit-learn__scikit-learn-14141", "repo": "scikit-learn/scikit-learn"}, + {"instance_id": "astropy__astropy-7166", "repo": "astropy/astropy"}, + {"instance_id": "pallets__flask-5014", "repo": "pallets/flask"} + ] +} diff --git a/benchmarks/code/subsets/swebench-verified-random50.json b/benchmarks/code/subsets/swebench-verified-random50.json new file mode 100644 index 0000000..5ac29ca --- /dev/null +++ b/benchmarks/code/subsets/swebench-verified-random50.json @@ -0,0 +1,217 @@ +{ + "schema": "code-bench-subset/1.0", + "suite_id": "swebench-verified-random50", + "sampling": { + "method": "uniform-random-without-replacement", + "seed": 20260901, + "drawn_from": { + "dataset": "princeton-nlp/SWE-bench_Verified", + "revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a", + "split": "test", + "population": 500 + }, + "note": "A uniform random draw, so the observed score is an unbiased estimate of the full-split score with a binomial interval." + }, + "instances": [ + { + "instance_id": "astropy__astropy-12907", + "repo": "astropy/astropy" + }, + { + "instance_id": "astropy__astropy-13033", + "repo": "astropy/astropy" + }, + { + "instance_id": "astropy__astropy-13398", + "repo": "astropy/astropy" + }, + { + "instance_id": "astropy__astropy-14365", + "repo": "astropy/astropy" + }, + { + "instance_id": "django__django-11163", + "repo": "django/django" + }, + { + "instance_id": "django__django-11179", + "repo": "django/django" + }, + { + "instance_id": "django__django-11451", + "repo": "django/django" + }, + { + "instance_id": "django__django-11477", + "repo": "django/django" + }, + { + "instance_id": "django__django-12754", + "repo": "django/django" + }, + { + "instance_id": "django__django-13128", + "repo": "django/django" + }, + { + "instance_id": "django__django-13807", + "repo": "django/django" + }, + { + "instance_id": "django__django-14053", + "repo": "django/django" + }, + { + "instance_id": "django__django-14315", + "repo": "django/django" + }, + { + "instance_id": "django__django-14534", + "repo": "django/django" + }, + { + "instance_id": "django__django-14631", + "repo": "django/django" + }, + { + "instance_id": "django__django-14792", + "repo": "django/django" + }, + { + "instance_id": "django__django-15022", + "repo": "django/django" + }, + { + "instance_id": "django__django-15280", + "repo": "django/django" + }, + { + "instance_id": "django__django-15380", + "repo": "django/django" + }, + { + "instance_id": "django__django-15525", + "repo": "django/django" + }, + { + "instance_id": "django__django-15731", + "repo": "django/django" + }, + { + "instance_id": "django__django-16100", + "repo": "django/django" + }, + { + "instance_id": "django__django-16263", + "repo": "django/django" + }, + { + "instance_id": "django__django-16493", + "repo": "django/django" + }, + { + "instance_id": "django__django-16612", + "repo": "django/django" + }, + { + "instance_id": "django__django-16819", + "repo": "django/django" + }, + { + "instance_id": "matplotlib__matplotlib-21568", + "repo": "matplotlib/matplotlib" + }, + { + "instance_id": "matplotlib__matplotlib-24970", + "repo": "matplotlib/matplotlib" + }, + { + "instance_id": "mwaskom__seaborn-3069", + "repo": "mwaskom/seaborn" + }, + { + "instance_id": "psf__requests-6028", + "repo": "psf/requests" + }, + { + "instance_id": "pydata__xarray-4075", + "repo": "pydata/xarray" + }, + { + "instance_id": "pydata__xarray-4687", + "repo": "pydata/xarray" + }, + { + "instance_id": "pydata__xarray-6599", + "repo": "pydata/xarray" + }, + { + "instance_id": "pydata__xarray-6721", + "repo": "pydata/xarray" + }, + { + "instance_id": "pydata__xarray-6992", + "repo": "pydata/xarray" + }, + { + "instance_id": "pytest-dev__pytest-7571", + "repo": "pytest-dev/pytest" + }, + { + "instance_id": "scikit-learn__scikit-learn-12682", + "repo": "scikit-learn/scikit-learn" + }, + { + "instance_id": "scikit-learn__scikit-learn-13142", + "repo": "scikit-learn/scikit-learn" + }, + { + "instance_id": "scikit-learn__scikit-learn-13496", + "repo": "scikit-learn/scikit-learn" + }, + { + "instance_id": "scikit-learn__scikit-learn-14983", + "repo": "scikit-learn/scikit-learn" + }, + { + "instance_id": "sphinx-doc__sphinx-10614", + "repo": "sphinx-doc/sphinx" + }, + { + "instance_id": "sphinx-doc__sphinx-8120", + "repo": "sphinx-doc/sphinx" + }, + { + "instance_id": "sphinx-doc__sphinx-8269", + "repo": "sphinx-doc/sphinx" + }, + { + "instance_id": "sphinx-doc__sphinx-8621", + "repo": "sphinx-doc/sphinx" + }, + { + "instance_id": "sphinx-doc__sphinx-8638", + "repo": "sphinx-doc/sphinx" + }, + { + "instance_id": "sympy__sympy-12419", + "repo": "sympy/sympy" + }, + { + "instance_id": "sympy__sympy-12489", + "repo": "sympy/sympy" + }, + { + "instance_id": "sympy__sympy-13480", + "repo": "sympy/sympy" + }, + { + "instance_id": "sympy__sympy-16450", + "repo": "sympy/sympy" + }, + { + "instance_id": "sympy__sympy-23262", + "repo": "sympy/sympy" + } + ] +} diff --git a/benchmarks/code/suites.json b/benchmarks/code/suites.json new file mode 100644 index 0000000..5e258ee --- /dev/null +++ b/benchmarks/code/suites.json @@ -0,0 +1,32 @@ +{ + "schema": "code-bench-suites/1.0", + "suites": [ + { + "id": "swebench-verified-canary", + "dataset": "princeton-nlp/SWE-bench_Verified", + "split": "test", + "subset_file": "subsets/swebench-verified-canary.json", + "task_count": 5, + "default_conditions": [ + "A", + "B", + "C", + "D", + "E", + "F", + "G" + ], + "require_unique_repos": true + }, + { + "id": "swebench-verified-random50", + "dataset": "princeton-nlp/SWE-bench_Verified", + "split": "test", + "subset_file": "subsets/swebench-verified-random50.json", + "task_count": 50, + "default_conditions": [ + "B" + ] + } + ] +} diff --git a/benchmarks/code/tests/test-code-bench.sh b/benchmarks/code/tests/test-code-bench.sh new file mode 100644 index 0000000..ba00cfc --- /dev/null +++ b/benchmarks/code/tests/test-code-bench.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +RUNNER="$CODE_DIR/code-bench.sh" +TMP=$(mktemp -d -t code-bench-test-XXXXXX) +trap 'rm -rf "$TMP"' EXIT + +TOTAL=0 +FAILED=0 +pass() { TOTAL=$((TOTAL + 1)); printf 'PASS: %s\n' "$1"; } +fail() { TOTAL=$((TOTAL + 1)); FAILED=$((FAILED + 1)); printf 'FAIL: %s\n' "$1"; } +expect_ok() { if "$@" >/dev/null 2>&1; then pass "$1"; else fail "$1"; fi; } + +if bash "$RUNNER" check >/dev/null 2>&1; then pass "manifests validate"; else fail "manifests validate"; fi + +estimate=$(bash "$RUNNER" estimate --suite swebench-verified-canary --conditions A,B,C --json 2>/dev/null) +if [[ "$(printf '%s' "$estimate" | jq -r '.cells')" == 15 \ + && "$(printf '%s' "$estimate" | jq -r '.declared_dispatches.claude')" == 20 \ + && "$(printf '%s' "$estimate" | jq -r '.declared_dispatches.codex')" == 10 \ + && "$(printf '%s' "$estimate" | jq -r '.declared_dispatches.glm')" == 5 \ + && "$(printf '%s' "$estimate" | jq -r '.declared_dispatches.kimi')" == 5 ]]; then + pass "A/B/C estimate is exact" +else + fail "A/B/C estimate is exact" +fi + +rc=0 +bash "$RUNNER" estimate --conditions A,B,C --max-claude-dispatches 4 >/dev/null 2>&1 || rc=$? +if [[ "$rc" == 75 ]]; then pass "Claude cap fails closed"; else fail "Claude cap fails closed (rc=$rc)"; fi + +cat > "$TMP/good.jsonl" <<'JSON' +{"instance_id":"sympy__sympy-20916","model_name_or_path":"condition-A","model_patch":"diff --git a/a.py b/a.py\n"} +JSON +if bash "$RUNNER" validate-predictions "$TMP/good.jsonl" >/dev/null 2>&1; then pass "valid prediction accepted"; else fail "valid prediction accepted"; fi + +cat > "$TMP/unknown.jsonl" <<'JSON' +{"instance_id":"unknown__repo-1","model_name_or_path":"condition-A","model_patch":"diff --git a/a b/a\n"} +JSON +if bash "$RUNNER" validate-predictions "$TMP/unknown.jsonl" >/dev/null 2>&1; then fail "unknown instance rejected"; else pass "unknown instance rejected"; fi + +cat > "$TMP/duplicate.jsonl" <<'JSON' +{"instance_id":"sympy__sympy-20916","model_name_or_path":"condition-A","model_patch":"diff one"} +{"instance_id":"sympy__sympy-20916","model_name_or_path":"condition-A","model_patch":"diff two"} +JSON +if bash "$RUNNER" validate-predictions "$TMP/duplicate.jsonl" >/dev/null 2>&1; then fail "duplicate prediction rejected"; else pass "duplicate prediction rejected"; fi + +cat > "$TMP/empty.jsonl" <<'JSON' +{"instance_id":"sympy__sympy-20916","model_name_or_path":"condition-A","model_patch":""} +JSON +if bash "$RUNNER" validate-predictions "$TMP/empty.jsonl" >/dev/null 2>&1; then fail "empty patch rejected"; else pass "empty patch rejected"; fi + +if grep -R -nE '"(patch|test_patch|FAIL_TO_PASS|PASS_TO_PASS)"[[:space:]]*:' \ + "$CODE_DIR/subsets" "$CODE_DIR/conditions.json" "$CODE_DIR/suites.json" >/dev/null 2>&1; then + fail "checked-in manifests contain no gold fields" +else + pass "checked-in manifests contain no gold fields" +fi + +if find "$CODE_DIR" -type f -name '*.sh' -exec grep -nE \ + '^[[:space:]]*(mapfile|readarray)([[:space:]]|$)' {} + >/dev/null 2>&1; then + fail "shell scripts are Bash 3 portable" +else + pass "shell scripts are Bash 3 portable" +fi + +TEST_RESULTS="$TMP/results" +for condition in A B C D E F; do + cell="$TEST_RESULTS/runs/test/sympy__sympy-20916/$condition" + mkdir -p "$cell/workspace/.git" "$TEST_RESULTS/predictions/test" + printf 'task\n' > "$cell/task.md" + jq -n --arg c "$condition" --arg w "$cell/workspace" \ + --arg t "$cell/task.md" \ + '{instance_id:"sympy__sympy-20916",condition:$c,workspace:$w,task_file:$t}' \ + > "$cell/input.json" +done + +dry_c=$(CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-workflow \ + --input "$TEST_RESULTS/runs/test/sympy__sympy-20916/C/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/C.jsonl" \ + --max-claude-dispatches 2 --dry-run 2>/dev/null) +if [[ "$(printf '%s' "$dry_c" | jq -r '.executed')" == false \ + && "$(printf '%s' "$dry_c" | jq -r '.phases | length')" == 5 \ + && "$(printf '%s' "$dry_c" | jq -r '.declared_claude_dispatches')" == 2 ]]; then + pass "condition C dry-run exposes five phases and executes nothing" +else + fail "condition C dry-run exposes five phases and executes nothing" +fi + +rc=0 +CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-workflow \ + --input "$TEST_RESULTS/runs/test/sympy__sympy-20916/D/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/D.jsonl" \ + --max-claude-dispatches 1 --dry-run >/dev/null 2>&1 || rc=$? +if [[ "$rc" == 75 ]]; then pass "workflow cap refuses condition D"; else fail "workflow cap refuses condition D (rc=$rc)"; fi + +mkdir -p "$TMP/outside/workspace/.git" +printf 'task\n' > "$TMP/outside/task.md" +jq -n --arg w "$TMP/outside/workspace" --arg t "$TMP/outside/task.md" \ + '{instance_id:"sympy__sympy-20916",condition:"A",workspace:$w,task_file:$t}' > "$TMP/outside/input.json" +if CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-workflow --input "$TMP/outside/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/A.jsonl" --max-claude-dispatches 1 --dry-run >/dev/null 2>&1; then + fail "driver rejects input outside sandbox" +else + pass "driver rejects input outside sandbox" +fi + +if bash "$RUNNER" run-canary --run-id dry-one --conditions A,B,C --task-limit 1 \ + --max-claude-dispatches 4 --dry-run >/dev/null 2>&1; then + pass "one-task A/B/C canary fits aggregate cap four" +else + fail "one-task A/B/C canary fits aggregate cap four" +fi + +rc=0 +bash "$RUNNER" run-canary --run-id dry-two --conditions A,B,C --task-limit 2 \ + --max-claude-dispatches 4 --dry-run >/dev/null 2>&1 || rc=$? +if [[ "$rc" == 75 ]]; then pass "two-task A/B/C canary exceeds aggregate cap four"; else fail "two-task A/B/C canary exceeds aggregate cap four (rc=$rc)"; fi + +if jq -e 'all(.conditions[]; (.tier == "agentic") or (.tier == "single-shot"))' \ + "$CODE_DIR/conditions.json" >/dev/null 2>&1; then + pass "every condition declares a tier" +else + fail "every condition declares a tier" +fi + +if jq -e '[.conditions[] | select(.tier == "single-shot")] + | length > 0 and all(.[]; .dispatches.claude == 0 and .dispatches.codex == 0)' \ + "$CODE_DIR/conditions.json" >/dev/null 2>&1; then + pass "single-shot conditions spend no agentic dispatch" +else + fail "single-shot conditions spend no agentic dispatch" +fi + +refusal=$(CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-workflow \ + --input "$TEST_RESULTS/runs/test/sympy__sympy-20916/F/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/F.jsonl" \ + --max-claude-dispatches 0 --dry-run 2>&1 >/dev/null) +rc=$? +if (( rc != 0 )) && printf '%s' "$refusal" | grep -q 'run-single-shot.sh'; then + pass "agentic driver refuses a single-shot condition" +else + fail "agentic driver refuses a single-shot condition (rc=$rc)" +fi + +dry_f=$(CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-single-shot \ + --input "$TEST_RESULTS/runs/test/sympy__sympy-20916/F/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/F.jsonl" \ + --agent glm --dry-run 2>/dev/null) +if [[ "$(printf '%s' "$dry_f" | jq -r '.tier')" == "single-shot" \ + && "$(printf '%s' "$dry_f" | jq -r '.executed')" == false ]]; then + pass "single-shot dry-run reports its tier and executes nothing" +else + fail "single-shot dry-run reports its tier and executes nothing" +fi + +if CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-single-shot \ + --input "$TEST_RESULTS/runs/test/sympy__sympy-20916/F/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/F.jsonl" \ + --agent kimi --dry-run >/dev/null 2>&1; then + fail "single-shot driver rejects an undeclared agent" +else + pass "single-shot driver rejects an undeclared agent" +fi + +if CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-single-shot \ + --input "$TEST_RESULTS/runs/test/sympy__sympy-20916/A/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/A.jsonl" \ + --agent glm --dry-run >/dev/null 2>&1; then + fail "single-shot driver rejects an agentic condition" +else + pass "single-shot driver rejects an agentic condition" +fi + +EXTRACT="$CODE_DIR/scripts/extract-diff.sh" +FIX="$TMP/fixture" +mkdir -p "$FIX" +git -C "$FIX" init -q +printf 'alpha\nbeta\ngamma\n' > "$FIX/sample.txt" +git -C "$FIX" add sample.txt >/dev/null 2>&1 +git -C "$FIX" -c user.email=t@e -c user.name=t commit -qm seed >/dev/null 2>&1 +printf 'alpha\nBETA\ngamma\n' > "$FIX/sample.txt" +git -C "$FIX" diff > "$TMP/real.patch" +git -C "$FIX" checkout -- sample.txt + +{ + printf 'Here is the fix you asked for.\n\n' + printf '%s\n' '```diff' + cat "$TMP/real.patch" + printf '%s\n' '```' + printf '\nLet me know if you want tests as well.\n' +} > "$TMP/fenced-response.md" +if bash "$EXTRACT" "$TMP/fenced-response.md" "$TMP/fenced.patch" >/dev/null 2>&1 \ + && git -C "$FIX" apply --check "$TMP/fenced.patch" >/dev/null 2>&1; then + pass "fenced diff extracts and applies" +else + fail "fenced diff extracts and applies" +fi + +{ + cat "$TMP/real.patch" + printf 'That should resolve the reported behaviour.\n' +} > "$TMP/bare-response.md" +if bash "$EXTRACT" "$TMP/bare-response.md" "$TMP/bare.patch" >/dev/null 2>&1 \ + && git -C "$FIX" apply --check "$TMP/bare.patch" >/dev/null 2>&1 \ + && ! grep -q 'reported behaviour' "$TMP/bare.patch"; then + pass "unfenced diff extracts without trailing prose" +else + fail "unfenced diff extracts without trailing prose" +fi + +printf 'I could not reproduce the issue.\n' > "$TMP/no-diff.md" +if bash "$EXTRACT" "$TMP/no-diff.md" "$TMP/none.patch" >/dev/null 2>&1; then + fail "response without a diff is rejected" +else + pass "response without a diff is rejected" +fi + +# The single-shot gate applies with --recount because chat models routinely get +# the @@ line counts wrong while proposing a correct edit. Strict apply rejects +# such a patch; --recount accepts it without altering a single edited line. +miscounted="$TMP/miscounted.patch" +sed 's/^@@ -1,3 +1,3 @@/@@ -1,9 +1,9 @@/' "$TMP/real.patch" > "$miscounted" +strict_rc=0 +git -C "$FIX" apply --check "$miscounted" >/dev/null 2>&1 || strict_rc=$? +recount_rc=0 +git -C "$FIX" apply --check --recount "$miscounted" >/dev/null 2>&1 || recount_rc=$? +if (( strict_rc != 0 )) && (( recount_rc == 0 )); then + pass "--recount rescues a miscounted hunk header" +else + fail "--recount rescues a miscounted hunk header (strict=$strict_rc recount=$recount_rc)" +fi + +if grep -q 'apply --check --recount' "$CODE_DIR/drivers/run-single-shot.sh"; then + pass "single-shot driver gates with --recount" +else + fail "single-shot driver gates with --recount" +fi + +STATUS_TEST="$TMP/status/battery.txt" +if ( set -e + source "$CODE_DIR/lib/code-bench-lib.sh" + code_status_init "$STATUS_TEST" alpha + code_status_append "$STATUS_TEST" alpha "cells=1/5" + ) >/dev/null 2>&1 && grep -q '^writer=alpha .*cells=1/5' "$STATUS_TEST"; then + pass "status lines carry their writer" +else + fail "status lines carry their writer" +fi + +if ( source "$CODE_DIR/lib/code-bench-lib.sh" + code_status_append "$STATUS_TEST" beta "cells=2/5" + ) >/dev/null 2>&1; then + fail "a second writer cannot append to another writer's status file" +else + pass "a second writer cannot append to another writer's status file" +fi + +sandbox_default=$( source "$CODE_DIR/lib/code-bench-lib.sh"; code_codex_sandbox ) +if [[ "$sandbox_default" == "workspace-write" ]]; then + pass "codex sandbox defaults to workspace-write" +else + fail "codex sandbox defaults to workspace-write" +fi + +if ( source "$CODE_DIR/lib/code-bench-lib.sh" + CODE_BENCH_CODEX_SANDBOX=wide-open code_codex_sandbox + ) >/dev/null 2>&1; then + fail "codex sandbox rejects an unknown mode" +else + pass "codex sandbox rejects an unknown mode" +fi + +printf '%d/%d assertions passed' "$((TOTAL - FAILED))" "$TOTAL" +if (( FAILED > 0 )); then printf ' (%d failed)\n' "$FAILED"; exit 1; fi +printf '\n' diff --git a/benchmarks/code/validate-predictions.sh b/benchmarks/code/validate-predictions.sh new file mode 100644 index 0000000..20379b9 --- /dev/null +++ b/benchmarks/code/validate-predictions.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/code-bench-lib.sh +source "$SCRIPT_DIR/lib/code-bench-lib.sh" + +PREDICTIONS="${1:-}" +SUITE="${2:-swebench-verified-canary}" +[[ -n "$PREDICTIONS" && -f "$PREDICTIONS" ]] || { code_die "usage: validate-predictions.sh FILE [SUITE]"; exit 2; } + +suite_json=$(code_suite_json "$SUITE") || { code_die "unknown suite: $SUITE"; exit 1; } +subset=$(code_subset_path "$suite_json") +seen=$(mktemp -t code-bench-seen-XXXXXX) +trap 'rm -f "$seen"' EXIT +line_no=0 +records=0 + +while IFS= read -r line || [[ -n "$line" ]]; do + line_no=$((line_no + 1)) + [[ -n "${line//[[:space:]]/}" ]] || continue + if ! printf '%s' "$line" | jq -e ' + type == "object" and + (.instance_id | type == "string" and length > 0) and + (.model_name_or_path | type == "string" and length > 0) and + (.model_patch | type == "string" and startswith("diff --git ")) + ' >/dev/null 2>&1; then + code_die "invalid prediction record at line $line_no"; exit 1 + fi + instance=$(printf '%s' "$line" | jq -r '.instance_id' | tr -d '\r') + jq -e --arg id "$instance" '.instances[] | select(.instance_id == $id)' "$subset" >/dev/null \ + || { code_die "prediction line $line_no names an instance outside suite $SUITE: $instance"; exit 1; } + if grep -qxF "$instance" "$seen"; then + code_die "duplicate prediction for $instance"; exit 1 + fi + printf '%s\n' "$instance" >> "$seen" + records=$((records + 1)) +done < "$PREDICTIONS" + +(( records > 0 )) || { code_die "prediction file contains no records"; exit 1; } +printf 'VALID: %s prediction record(s) for %s\n' "$records" "$SUITE" diff --git a/benchmarks/site/aggregate.sh b/benchmarks/site/aggregate.sh new file mode 100644 index 0000000..e1b2dda --- /dev/null +++ b/benchmarks/site/aggregate.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Build the results site's data file from evaluator output and run logs. +# +# The published page renders this JSON and nothing else, so anything that is +# not traceable to a file on disk cannot reach the page. Standardized +# benchmarks only: the retired document suite is not a source here. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +RESULTS_ROOT="${CODE_BENCH_RESULTS_ROOT:-$REPO_ROOT/benchmarks/results/code}" +SUITE="swebench-verified-canary" +OUTPUT="$RESULTS_ROOT/site/leaderboard.json" + +while (( $# > 0 )); do + case "$1" in + --suite) SUITE="${2:?--suite needs a value}"; shift 2 ;; + --output) OUTPUT="${2:?--output needs a value}"; shift 2 ;; + *) printf 'ERROR: unknown aggregate option: %s\n' "$1" >&2; exit 2 ;; + esac +done + +command -v python >/dev/null 2>&1 || { printf 'ERROR: python is required\n' >&2; exit 1; } +[[ -d "$RESULTS_ROOT/evaluation" ]] || { printf 'ERROR: no evaluation directory under %s\n' "$RESULTS_ROOT" >&2; exit 1; } + +python "$SCRIPT_DIR/build-site-data.py" \ + --repo-root "$REPO_ROOT" \ + --results-root "$RESULTS_ROOT" \ + --suite "$SUITE" \ + --output "$OUTPUT" \ + --generated-at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +python "$SCRIPT_DIR/render-page.py" --data "$OUTPUT" --output "${OUTPUT%%.json}.html" diff --git a/benchmarks/site/build-site-data.py b/benchmarks/site/build-site-data.py new file mode 100644 index 0000000..64d4e95 --- /dev/null +++ b/benchmarks/site/build-site-data.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 +"""Build the single JSON the results page renders. + +Every field here is read from a file on disk and carries the path it came +from, so a reader can check any number on the page against the evaluator's +own output. Nothing is entered by hand and nothing is inferred. + +Sources: + benchmarks/code/conditions.json condition ids, labels, tiers + benchmarks/code/suites.json suite and subset pointer + benchmarks/code/subsets/*.json the frozen task list + benchmarks/code/external-sources.lock.json evaluator and dataset pins + results/code/evaluation/*.json official evaluator reports + results/code/evaluation/logs/... per-instance evaluator reports + results/code/runs/*/*// run manifests and provider logs + +Only standardized-benchmark material is read. The retired document suite under +benchmarks/ is not a source here and must never become one. +""" +import argparse +import glob +import json +import os +import re +import subprocess +import sys + +REPORT_NAME_RE = re.compile( + r'^co-evolution-condition-(?P[A-Za-z0-9_-]+)\.(?P.+)\.json$') +GOLD_NAME_RE = re.compile(r'^gold\.(?P.+)\.json$') + + +def read_json(path): + with open(path, encoding='utf-8') as handle: + return json.load(handle) + + +def rel(root, path): + return os.path.relpath(path, root).replace(os.sep, '/') + + +def harness_commit(root): + proc = subprocess.run(('git', '-C', root, 'rev-parse', 'HEAD'), + capture_output=True, text=True) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def harness_dirty(root): + proc = subprocess.run(('git', '-C', root, 'status', '--porcelain'), + capture_output=True, text=True) + return bool(proc.stdout.strip()) if proc.returncode == 0 else None + + +def newest_reports(eval_dir): + """Latest evaluator report per condition, plus the ones it supersedes.""" + latest, superseded = {}, [] + for path in sorted(glob.glob(os.path.join(eval_dir, '*.json'))): + match = REPORT_NAME_RE.match(os.path.basename(path)) + if not match: + continue + cond, run_id = match.group('cond'), match.group('run_id') + previous = latest.get(cond) + if previous is None or run_id > previous[0]: + if previous is not None: + superseded.append((cond, previous[0], previous[1])) + latest[cond] = (run_id, path) + else: + superseded.append((cond, run_id, path)) + return latest, superseded + + +def gold_canary(eval_dir): + best = None + for path in sorted(glob.glob(os.path.join(eval_dir, 'gold.*.json'))): + match = GOLD_NAME_RE.match(os.path.basename(path)) + if not match: + continue + if best is None or match.group('run_id') > best[0]: + best = (match.group('run_id'), path) + if best is None: + return None + data = read_json(best[1]) + return { + 'run_id': best[0], + 'report_file': best[1], + 'submitted': data.get('submitted_instances'), + 'resolved': data.get('resolved_instances'), + } + + +def per_instance_reports(eval_dir, run_id, model_name): + """Read the evaluator's own per-instance verdicts for one scored run.""" + base = os.path.join(eval_dir, 'logs', 'run_evaluation', run_id, model_name) + verdicts = {} + if not os.path.isdir(base): + return verdicts + for instance in sorted(os.listdir(base)): + report = os.path.join(base, instance, 'report.json') + if not os.path.isfile(report): + continue + try: + payload = read_json(report) + except ValueError: + continue + entry = payload.get(instance) or {} + tests = entry.get('tests_status') or {} + f2p = tests.get('FAIL_TO_PASS') or {} + p2p = tests.get('PASS_TO_PASS') or {} + verdicts[instance] = { + 'resolved': bool(entry.get('resolved')), + 'patch_applied': bool(entry.get('patch_successfully_applied')), + 'fail_to_pass_passed': len(f2p.get('success') or []), + 'fail_to_pass_failed': len(f2p.get('failure') or []), + 'pass_to_pass_failed': len(p2p.get('failure') or []), + 'report_file': report, + } + return verdicts + + +def scored_patches(eval_dir, run_id, model_name): + """The exact patch text the evaluator scored, per instance.""" + base = os.path.join(eval_dir, 'logs', 'run_evaluation', run_id, model_name) + patches = {} + if not os.path.isdir(base): + return patches + for instance in sorted(os.listdir(base)): + diff = os.path.join(base, instance, 'patch.diff') + if os.path.isfile(diff): + with open(diff, encoding='utf-8', errors='replace') as handle: + patches[instance] = handle.read() + return patches + + +def index_cells(runs_root): + """Map (condition, instance, patch text) -> cell directory.""" + index = {} + pattern = os.path.join(runs_root, '*', '*', '*', 'prediction.json') + for path in glob.glob(pattern): + try: + record = read_json(path) + except ValueError: + continue + model = record.get('model_name_or_path', '') + if not model.startswith('co-evolution-condition-'): + continue + key = (model[len('co-evolution-condition-'):], + record.get('instance_id'), + (record.get('model_patch') or '').strip()) + index[key] = os.path.dirname(path) + return index + + +def index_attempts(runs_root): + """Cells that ran and recorded an outcome, keyed by (condition, instance). + + A single-shot cell that never produced an applicable patch writes an + outcome.json and no prediction. Without this index such a cell is + indistinguishable from one that was never run, which would read on the page + as missing data rather than as the failure it is. + """ + attempts = {} + for path in glob.glob(os.path.join(runs_root, '*', '*', '*', 'outcome.json')): + try: + record = read_json(path) + except ValueError: + continue + key = (record.get('condition'), record.get('instance')) + attempts[key] = { + 'outcome': record.get('outcome'), + 'attempts': record.get('attempts'), + 'cell_dir': os.path.dirname(path), + } + return attempts + + +def cell_telemetry(cell_dir): + """Provider effort for one cell, from the CLI's own envelope figures.""" + out = { + 'cell_dir': cell_dir, + 'claude_dispatches': 0, + 'claude_cost_usd': 0.0, + 'claude_output_tokens': 0, + 'claude_wall_seconds': 0, + 'codex_phases': 0, + 'single_shot_attempts': None, + 'sandbox': None, + } + manifest = os.path.join(cell_dir, 'run-manifest.json') + if os.path.isfile(manifest): + try: + data = read_json(manifest) + out['sandbox'] = (data.get('sandbox') or {}).get('codex') + out['schema'] = data.get('schema') + except ValueError: + pass + outcome = os.path.join(cell_dir, 'outcome.json') + if os.path.isfile(outcome): + try: + out['single_shot_attempts'] = read_json(outcome).get('attempts') + except ValueError: + pass + logs = os.path.join(cell_dir, 'logs') + if os.path.isdir(logs): + for name in sorted(os.listdir(logs)): + path = os.path.join(logs, name) + if name.startswith('fable-') and name.endswith('.json'): + try: + data = read_json(path) + except ValueError: + continue + out['claude_dispatches'] += 1 + out['claude_cost_usd'] += float(data.get('total_cost_usd') or 0) + usage = data.get('usage') or {} + out['claude_output_tokens'] += int(usage.get('output_tokens') or 0) + out['claude_wall_seconds'] += int((data.get('duration_ms') or 0) / 1000) + elif name.startswith('codex-') and name.endswith('.log'): + out['codex_phases'] += 1 + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--repo-root', required=True) + ap.add_argument('--results-root', required=True) + ap.add_argument('--suite', default='swebench-verified-canary') + ap.add_argument('--output', required=True) + ap.add_argument('--generated-at', required=True, + help='UTC timestamp supplied by the caller') + args = ap.parse_args() + + root = os.path.abspath(args.repo_root) + results = os.path.abspath(args.results_root) + code_dir = os.path.join(root, 'benchmarks', 'code') + eval_dir = os.path.join(results, 'evaluation') + runs_root = os.path.join(results, 'runs') + + conditions = read_json(os.path.join(code_dir, 'conditions.json'))['conditions'] + suites = read_json(os.path.join(code_dir, 'suites.json'))['suites'] + suite = next((s for s in suites if s['id'] == args.suite), None) + if suite is None: + sys.exit('ERROR: unknown suite %s' % args.suite) + subset_path = os.path.join(code_dir, suite['subset_file']) + subset = read_json(subset_path) + instances = [row['instance_id'] for row in subset['instances']] + repos = {row['instance_id']: row['repo'] for row in subset['instances']} + lock = read_json(os.path.join(code_dir, 'external-sources.lock.json')) + + latest, superseded = newest_reports(eval_dir) + cells = index_cells(runs_root) + attempts_index = index_attempts(runs_root) + + rows = [] + for condition in conditions: + cond_id = condition['id'] + row = { + 'condition': cond_id, + 'label': condition['label'], + 'tier': condition['tier'], + 'description': condition['description'], + 'declared_dispatches': condition['dispatches'], + 'measured': False, + 'attempted': False, + 'report_file': None, + 'evaluator_run_id': None, + 'submitted': 0, + 'resolved': 0, + 'per_task': [], + 'telemetry': { + 'claude_dispatches': 0, + 'claude_cost_usd': 0.0, + 'claude_output_tokens': 0, + 'claude_wall_seconds': 0, + 'codex_phases': 0, + 'cells_linked': 0, + 'sandbox_modes': [], + 'single_shot_attempts': [], + }, + } + entry = latest.get(cond_id) + if entry is not None: + run_id, report_path = entry + report = read_json(report_path) + model_name = 'co-evolution-condition-%s' % cond_id + verdicts = per_instance_reports(eval_dir, run_id, model_name) + patches = scored_patches(eval_dir, run_id, model_name) + row['measured'] = True + row['attempted'] = True + row['report_file'] = rel(root, report_path) + row['evaluator_run_id'] = run_id + row['submitted'] = report.get('submitted_instances') or 0 + row['resolved'] = report.get('resolved_instances') or 0 + row['dataset_total_instances'] = report.get('total_instances') + + sandboxes, attempts = set(), [] + for instance in instances: + verdict = verdicts.get(instance) + if verdict is None: + attempt = attempts_index.get((cond_id, instance)) + row['per_task'].append({ + 'instance_id': instance, 'repo': repos[instance], + 'status': 'no-patch' if attempt else 'not-submitted', + 'attempt_outcome': attempt['outcome'] if attempt else None, + 'attempts': attempt['attempts'] if attempt else None, + 'evidence': (rel(root, os.path.join(attempt['cell_dir'], 'outcome.json')) + if attempt else None), + }) + if attempt: + row['attempted'] = True + continue + cell = cells.get((cond_id, instance, + (patches.get(instance) or '').strip())) + task = { + 'instance_id': instance, + 'repo': repos[instance], + 'status': 'resolved' if verdict['resolved'] else 'unresolved', + 'patch_applied': verdict['patch_applied'], + 'fail_to_pass_passed': verdict['fail_to_pass_passed'], + 'fail_to_pass_failed': verdict['fail_to_pass_failed'], + 'pass_to_pass_failed': verdict['pass_to_pass_failed'], + 'evidence': rel(root, verdict['report_file']), + 'cell_dir': rel(root, cell) if cell else None, + } + row['per_task'].append(task) + if cell: + telemetry = cell_telemetry(cell) + row['telemetry']['cells_linked'] += 1 + row['telemetry']['claude_dispatches'] += telemetry['claude_dispatches'] + row['telemetry']['claude_cost_usd'] += telemetry['claude_cost_usd'] + row['telemetry']['claude_output_tokens'] += telemetry['claude_output_tokens'] + row['telemetry']['claude_wall_seconds'] += telemetry['claude_wall_seconds'] + row['telemetry']['codex_phases'] += telemetry['codex_phases'] + if telemetry['sandbox']: + sandboxes.add(telemetry['sandbox']) + if telemetry['single_shot_attempts'] is not None: + attempts.append(telemetry['single_shot_attempts']) + row['telemetry']['sandbox_modes'] = sorted(sandboxes) + row['telemetry']['single_shot_attempts'] = attempts + else: + for instance in instances: + attempt = attempts_index.get((cond_id, instance)) + if attempt: + row['attempted'] = True + row['per_task'].append({ + 'instance_id': instance, 'repo': repos[instance], + 'status': 'no-patch' if attempt else 'not-run', + 'attempt_outcome': attempt['outcome'] if attempt else None, + 'attempts': attempt['attempts'] if attempt else None, + 'evidence': (rel(root, os.path.join(attempt['cell_dir'], 'outcome.json')) + if attempt else None), + }) + row['telemetry']['claude_cost_usd'] = round(row['telemetry']['claude_cost_usd'], 4) + rows.append(row) + + payload = { + 'schema': 'code-bench-site/1.0', + 'generated_at': args.generated_at, + 'harness': { + 'repo_commit': harness_commit(root), + 'working_tree_dirty': harness_dirty(root), + 'swebench_commit': lock.get('swebench', {}).get('commit'), + 'dataset': suite.get('dataset'), + 'dataset_revision': lock.get('dataset', {}).get('revision'), + 'lock_file': rel(root, os.path.join(code_dir, 'external-sources.lock.json')), + }, + 'suite': { + 'id': suite['id'], + 'dataset': suite['dataset'], + 'split': suite['split'], + 'task_count': suite['task_count'], + 'subset_file': rel(root, subset_path), + 'instances': [{'instance_id': i, 'repo': repos[i]} for i in instances], + }, + 'gold_canary': gold_canary(eval_dir), + 'rows': rows, + 'superseded_reports': [ + {'condition': cond, 'evaluator_run_id': run_id, 'report_file': rel(root, path)} + for cond, run_id, path in sorted(superseded) + ], + 'caveat': ('Frozen five-task probe of SWE-bench Verified, scored by the ' + 'official evaluator. One task is 20 points; these numbers are ' + 'not comparable to published full-500 SWE-bench Verified scores.'), + } + + gold = payload['gold_canary'] + if gold: + gold['report_file'] = rel(root, gold['report_file']) + + os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) + with open(args.output, 'w', encoding='utf-8', newline='\n') as handle: + json.dump(payload, handle, indent=2, sort_keys=False) + handle.write('\n') + print(args.output) + + +if __name__ == '__main__': + main() diff --git a/benchmarks/site/render-page.py b/benchmarks/site/render-page.py new file mode 100644 index 0000000..416e87b --- /dev/null +++ b/benchmarks/site/render-page.py @@ -0,0 +1,678 @@ +#!/usr/bin/env python3 +"""Render the results page from the aggregator's JSON and nothing else. + +Every figure on the page is read out of leaderboard.json, which in turn records +the evaluator report or run log each number came from. Prose here is framing; +it never states a result the JSON does not contain. + +The agentic and single-shot tiers are rendered as separate tables on purpose. A +single-shot seat gets one prompt and one answer with no tools and no test run, +so its score is not comparable to a coding agent's and must never appear in the +same ranked list without the label. +""" +import argparse +import html +import json +import os + +TIER_COPY = { + 'agentic': ('Agentic tier', + 'A coding agent works in the repository: reads files, edits them, ' + 'runs tests, and iterates before the patch is taken.'), + 'single-shot': ('Single-shot tier', + 'One prompt, one answer. The model sees the issue and a fixed ' + 'set of retrieved files, returns a diff, and never runs a test ' + 'or looks again. Not comparable to an agentic score.'), +} + +COMPOSITION = { + 'A': 'fable implements', + 'B': 'fable implements → codex repairs', + 'C': 'fable implements → codex + glm + kimi critique → fable repairs', + 'D': 'fable implements → fable reviews and repairs', + 'E': 'codex implements', + 'F': 'glm answers once, from retrieved context', + 'G': 'kimi answers once, from retrieved context', +} + + +def esc(value): + return html.escape('' if value is None else str(value), quote=True) + + +def dots(per_task): + cells = [] + for task in per_task: + status = task['status'] + klass = {'resolved': 'p', 'unresolved': 'f'}.get(status, 'n') + cells.append('' + % (klass, esc(task['instance_id']), esc(status))) + return '
%s
' % ''.join(cells) + + +def score_cell(row, task_count): + if not row['measured']: + # An arm that ran and produced nothing scorable scores zero. Only an arm + # that never ran gets a blank. + if row.get('attempted'): + return ('' + '0%%0 / %d' + '' % task_count) + return 'not run' + # The denominator is the frozen subset, never the number of predictions the + # arm managed to submit. A cell that produced no applicable patch failed its + # task; dropping it from the denominator would flatter the arm. + total = task_count + resolved = row['resolved'] or 0 + pct = (100.0 * resolved / total) if total else 0.0 + remaining = max(0.0, 100.0 - pct) + return ( + '' + '' + '%d%%' + '%d / %d' + ) % (pct, remaining, round(pct), resolved, total) + + +def coverage_chip(row, task_count): + if not row['measured']: + if row.get('attempted'): + return 'ran, no scorable patch' + return 'no data' + linked = row['telemetry']['cells_linked'] + submitted = row['submitted'] or 0 + if submitted < task_count: + return ('%d of %d patches submitted' + % (submitted, task_count)) + if linked < submitted: + return 'telemetry partial' + return 'fully measured' + + +def num_cell(value, sort_value=None, suffix=''): + if value is None: + return '—' + sort_value = value if sort_value is None else sort_value + return '%s%s' % (sort_value, value, suffix) + + +def leaderboard_table(rows, table_id, task_count): + body = [] + for row in rows: + klass = '' if (row['measured'] or row.get('attempted')) else ' class="absent"' + telemetry = row['telemetry'] + claude_calls = telemetry['claude_dispatches'] + seconds = telemetry['claude_wall_seconds'] + cost = telemetry['claude_cost_usd'] + codex_phases = telemetry['codex_phases'] + if row['measured'] or row.get('attempted'): + calls_cell = num_cell(claude_calls if claude_calls else 0) + time_cell = ('%s s' % (seconds, f'{seconds:,}') + if seconds else '—') + cost_cell = ('$%.2f' % (cost, cost) + if cost else 'no CLI figure') + codex_cell = num_cell(codex_phases if codex_phases else 0) + else: + calls_cell = time_cell = cost_cell = codex_cell = '—' + body.append( + '
' + '%s · %s' + '%s
%s%s%s%s%s%s%s' + % (klass, esc(row['condition']), esc(row['label']), + esc(COMPOSITION.get(row['condition'], row['description'])), + score_cell(row, task_count), dots(row['per_task']), + calls_cell, codex_cell, time_cell, cost_cell, coverage_chip(row, task_count))) + return ( + '
' + '' + '' + '' + '' + '' + '' + '' + '' + '%s
PipelineResolvedPer taskFable callsCodex phasesFable timeReported costCoverage
' + ) % (esc(table_id), ''.join(body)) + + +def task_matrix(data): + measured = [row for row in data['rows'] + if row['measured'] or row.get('attempted')] + instances = [entry['instance_id'] for entry in data['suite']['instances']] + header = ''.join('%s%s' + % (esc(row['condition']), + ' single-shot' if row['tier'] == 'single-shot' else '') + for row in measured) + body = [] + for instance in instances: + cells = [] + for row in measured: + task = next((t for t in row['per_task'] if t['instance_id'] == instance), None) + if task is None or task['status'] in ('not-submitted', 'not-run'): + cells.append('not scored') + continue + if task['status'] == 'no-patch': + cells.append('no patch' + % (esc(task.get('attempt_outcome') or 'no applicable patch'), + esc(task.get('attempts')))) + continue + if task['status'] == 'resolved': + cells.append('resolved') + continue + detail = [] + if task.get('fail_to_pass_failed'): + detail.append('%d F2P fail' % task['fail_to_pass_failed']) + if task.get('pass_to_pass_failed'): + detail.append('%d P2P fail' % task['pass_to_pass_failed']) + if not task.get('patch_applied'): + detail.append('patch did not apply') + cells.append('unresolved' + % esc(', '.join(detail) or 'unresolved')) + repo = next(e['repo'] for e in data['suite']['instances'] + if e['instance_id'] == instance) + body.append('%s%s%s' + % (esc(instance), esc(repo), ''.join(cells))) + return ( + '
' + '%s%s
InstanceRepository
' + ) % (header, ''.join(body)) + + +def provenance_table(data): + body = [] + for row in data['rows']: + if not row['measured']: + body.append('%sno evaluator report' + % esc(row['condition'])) + continue + body.append('%s%s%s' + % (esc(row['condition']), esc(row['evaluator_run_id']), esc(row['report_file']))) + for entry in data.get('superseded_reports', []): + body.append('%s%s' + '%s superseded' + % (esc(entry['condition']), esc(entry['evaluator_run_id']), + esc(entry['report_file']))) + return ('
' + '' + '%s
ConditionEvaluator runReport file
') % ''.join(body) + + +def tiles(data): + rows = data['rows'] + measured = [r for r in rows if r['measured'] or r.get('attempted')] + scored_cells = sum(r['submitted'] or 0 for r in measured) + claude_calls = sum(r['telemetry']['claude_dispatches'] for r in measured) + claude_seconds = sum(r['telemetry']['claude_wall_seconds'] for r in measured) + cost = sum(r['telemetry']['claude_cost_usd'] for r in measured) + items = [ + ('Configurations', '%d / %d' % (len(measured), len(rows)), + 'measured on this subset'), + ('Tasks per cell', str(data['suite']['task_count']), 'frozen subset'), + ('Scored cells', str(scored_cells), 'official evaluator, Docker'), + ('Fable dispatches', str(claude_calls), '%s s of model time' % f'{claude_seconds:,}'), + ('Reported cost', '$%.2f' % cost, 'CLI figure, Max plan'), + ] + return '
%s
' % ''.join( + '
%s%s' + '%s
' % (esc(k), v, esc(n)) for k, v, n in items) + + +STYLE = """ + + + + +""" + +SORT_SCRIPT = """ + +""" + + +# The narrative below describes one specific pattern in the results. It is +# guarded: if a later run stops producing that pattern, the section renders +# nothing rather than describing a run that did not happen. +SEPARATOR_TASK = 'sympy__sympy-20916' +UNICODE_DIGITS = ('B', 'D') +ASCII_DIGITS = ('A', 'C', 'E') + + +def status_on(rows, condition, instance): + row = next((r for r in rows if r['condition'] == condition), None) + if row is None: + return None + task = next((t for t in row['per_task'] if t['instance_id'] == instance), None) + return task['status'] if task else None + + +def findings(data): + rows = data['rows'] + resolved_ok = all(status_on(rows, c, SEPARATOR_TASK) == 'resolved' for c in UNICODE_DIGITS) + failed_ok = all(status_on(rows, c, SEPARATOR_TASK) == 'unresolved' for c in ASCII_DIGITS) + others_uniform = True + for entry in data['suite']['instances']: + if entry['instance_id'] == SEPARATOR_TASK: + continue + for cond in UNICODE_DIGITS + ASCII_DIGITS: + if status_on(rows, cond, entry['instance_id']) != 'resolved': + others_uniform = False + if not (resolved_ok and failed_ok and others_uniform): + return '' + + return ( + '

What actually separated the pipelines

' + '

Four of the five tasks resolved under every agentic ' + 'configuration. The entire spread across the leaderboard comes from one task, ' + 'and within that task from one character class.

' + '
' + '
' + 'Finding 01 · ' + esc(SEPARATOR_TASK) + '' + '

Every agentic arm wrote the same fix; two chose Unicode digits and passed

' + '

All five agentic configurations edited the same line of ' + 'sympy/printing/conventions.py, replacing an ASCII-only ' + 'name pattern with a Unicode-aware one. They differ in the second capture group ' + 'alone.

' + '
baseline '
+        "^([a-zA-Z]+)([0-9]+)$\n"
+        'B, D     '
+        "^([^" + chr(92) + "W" + chr(92) + "d_]+)(" + chr(92) + "d+)$     "
+        'Unicode digits → resolved\n'
+        'A, C, E  '
+        "^([^" + chr(92) + "W" + chr(92) + "d_]+)([0-9]+)$  "
+        'ASCII only → unresolved
' + '

The upstream test subscripts a non-ASCII digit, so the ASCII-only class fails it. ' + 'In condition C the panel critique argued explicitly that ' + '' + chr(92) + 'd+ would wrongly capture Arabic-Indic digits; ' + 'the repair pass accepted that reasoning and narrowed the class. The argument was ' + 'careful and the conclusion was wrong.

' + '
' + '
' + 'Finding 02 · harness' + '

The cross-vendor repair arm had never actually run

' + '

Codex on this Windows host accepts ' + '--sandbox workspace-write and then reports ' + 'sandbox: read-only. Across the earlier B cells it wrote a ' + 'review and changed nothing, so that run\'s 5/5 was Fable\'s first draft with a ' + 'discarded review attached.

' + '

The arm now runs with elevated access inside throwaway clones, and the mode is ' + 'recorded per cell. Re-run with a repair step that can write, B resolves the separator ' + 'task. Its two earlier reports are listed as superseded below rather than deleted.

' + '
' + '
') + + +def build(data): + harness = data['harness'] + gold = data['gold_canary'] or {} + suite = data['suite'] + rows = data['rows'] + agentic = [r for r in rows if r['tier'] == 'agentic'] + single = [r for r in rows if r['tier'] == 'single-shot'] + measured_single = [r for r in single if r['measured'] or r.get('attempted')] + + parts = [] + parts.append('Co-Evolution Code Battery') + parts.append(STYLE) + parts.append('
') + + parts.append( + '
' + '
SWE-bench Verified · %d-task frozen subset' + 'Official pinned evaluator, Docker' + 'Built %s
' + '

Co-Evolution Code Battery

' + '

Does putting a second model in the loop produce better patches ' + 'than one model working alone? Seven configurations, five pinned SWE-bench Verified ' + 'tasks, every patch scored by the official evaluator in Docker.

' + '
fable @ mediumgpt-5.6-sol @ medium' + 'glm-5.3-flash @ effort:lowkimi-k3 @ thinking:off' + 'phase timeout 900sgold canary %s/%s' + 'harness %s
' + % (suite['task_count'], esc(data['generated_at']), + esc(gold.get('resolved')), esc(gold.get('submitted')), + esc((harness.get('repo_commit') or '')[:7]))) + + parts.append( + '

Read this before the table

' + '

Five tasks is a probe, not a ranking. One task is 20 points, so a one-task gap ' + 'between two rows is well inside what a five-task sample produces by chance. ' + 'These numbers are also not comparable to published full-500 SWE-bench Verified ' + 'scores: the subset is fixed and was chosen for the harness, not drawn at random.

' + '

The two tiers are listed separately because they are not the same test. An agentic ' + 'row had file tools and could run the test suite; a single-shot row got one prompt and ' + 'answered once.

') + + parts.append(tiles(data)) + + parts.append('

Leaderboard

') + for tier, group in (('agentic', agentic), ('single-shot', single)): + title, note = TIER_COPY[tier] + parts.append('

%s

%s

' % (esc(title), esc(note))) + parts.append(leaderboard_table(group, 'board-%s' % tier, suite['task_count'])) + parts.append( + '
' + ' resolved' + ' unresolved' + ' no patch scored' + 'Task order: %s
' + % esc(' · '.join(e['instance_id'].split('__')[0] for e in suite['instances']))) + parts.append('
') + + parts.append(findings(data)) + + parts.append( + '

Task by task

' + '

One column per measured configuration. Hover an unresolved cell ' + 'for the failing test counts the evaluator recorded.

%s
' % task_matrix(data)) + + integrity = [] + integrity.append( + '
  • Evaluator. Official SWE-bench harness pinned at ' + '%s, dataset %s at revision ' + '%s, both from %s.
    • ' + % (esc((harness.get('swebench_commit') or '')[:12]), esc(harness.get('dataset')), + esc((harness.get('dataset_revision') or '')[:12]), esc(harness.get('lock_file')))) + if gold: + integrity.append( + '
  • Gold canary. A gold patch resolved %s/%s before any generated ' + 'prediction was scored, so a failure here means the patch, not the harness. Report: ' + '%s.
    • ' + % (esc(gold.get('resolved')), esc(gold.get('submitted')), esc(gold.get('report_file')))) + integrity.append( + '
  • Cost is the CLI\'s own figure for work billed to a Max ' + 'subscription, not metered API spend. The Claude CLI exposes no plan meter, so there is ' + 'no percentage-of-plan number here. Codex, GLM and Kimi report no per-call cost, so their ' + 'rows show dispatch counts instead.
    • ') + if measured_single: + integrity.append( + '
  • The single-shot tier is a floor, not a model ceiling. GLM and ' + 'Kimi are reachable here only as chat completions. Their cells receive the issue plus ' + 'a deterministic file selection and return one diff, gated by ' + 'git apply --check --recount. A cell that never produced an ' + 'applicable patch contributes no prediction and still counts against the five tasks. ' + '--recount recomputes the hunk line counts and changes no ' + 'line of the proposed edit: without it the gate scores a chat model\'s ability to ' + 'count lines, which the agentic arms never have to do because they edit files ' + 'directly.
    • ') + by_mode = {} + for row in rows: + for mode in row['telemetry']['sandbox_modes']: + by_mode.setdefault(mode, []).append(row['condition']) + if by_mode: + detail = '; '.join('%s in %s' + % (esc(mode), esc(', '.join(sorted(conds)))) + for mode, conds in sorted(by_mode.items())) + integrity.append( + '
  • Codex sandbox. %s. Codex on Windows accepts ' + 'workspace-write and then runs read-only, which makes a ' + 'repair arm look like it ran while changing nothing, so the mode each cell used is ' + 'recorded in its run manifest. The arms that needed Codex to write ran with elevated ' + 'access inside throwaway clones.
    • ' % detail) + integrity.append( + '
  • Every number on this page comes from a file. The page is generated ' + 'from benchmarks/site/aggregate.sh output; each row names the ' + 'evaluator report it was read from and each task cell is backed by that run\'s per-instance ' + 'report.json.
    • ') + + parts.append('

    Methodology and integrity

    ' + '
      %s
    %s
    ' + % (''.join(integrity), provenance_table(data))) + + dirty = harness.get('working_tree_dirty') + parts.append( + '
    Generated from run artifacts in ' + 'benchmarks/results/code/ by ' + 'benchmarks/site/aggregate.sh.
    ' + '
    Repository co-evolution-runtime · harness commit %s%s
    ' + '
    %s
    ' + % (esc(harness.get('repo_commit')), + ' · working tree had uncommitted changes at build time' if dirty else '', + esc(data['caveat']))) + + parts.append('
    ') + parts.append(SORT_SCRIPT) + return '\n'.join(parts) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--data', required=True) + ap.add_argument('--output', required=True) + args = ap.parse_args() + with open(args.data, encoding='utf-8') as handle: + data = json.load(handle) + page = build(data) + page = page.replace('/dev/null 2>&1 || die "glm seat requires curl" command -v jq >/dev/null 2>&1 || die "glm seat requires jq" + # glm-5.3-flash always reasons and Z.AI bills reasoning against max_tokens, so + # a capped caller gets finish_reason="length" with an empty content string + # unless the reasoning is bounded too. Z.AI rejects "disabled" and "medium"; + # low, high, and max are the only accepted levels. Unset keeps the provider + # default, which is what the document seats have always sent. + reasoning_effort="${GLM_REASONING_EFFORT:-}" + case "$reasoning_effort" in + ""|low|high|max) ;; + *) die "glm seat: GLM_REASONING_EFFORT must be empty, low, high, or max" ;; + esac request_file=$(mktemp -t glm-request-XXXXXX.json) response_file=$(mktemp -t glm-response-XXXXXX.json) @@ -616,12 +626,16 @@ invoke_glm() ( # incompatible with this document seat and were observed replacing a valid # response with / metadata. Z.AI's documented # Chat Completions endpoint avoids that extra agent layer entirely. - jq -Rs --arg model "${GLM_MODEL:-glm-5.3-flash}" '{ + jq -Rs --arg model "${GLM_MODEL:-glm-5.3-flash}" --argjson max_tokens "${GLM_MAX_TOKENS:-0}" \ + --arg reasoning_effort "$reasoning_effort" '{ model: $model, messages: [{role: "user", content: .}], stream: false, temperature: 0 - }' "$prompt_file" > "$request_file" + } + + (if $max_tokens > 0 then {max_tokens: $max_tokens} else {} end) + + (if $reasoning_effort != "" then {reasoning_effort: $reasoning_effort} else {} end)' \ + "$prompt_file" > "$request_file" # Keep the bearer token out of argv and logs. curl reads it from a mode-600 # temporary config file that this adapter removes on every exit path. @@ -665,11 +679,23 @@ invoke_kimi() ( local output_file="$2" local stderr_file="$3" local request_file response_file config_file - local curl_rc=0 message + local curl_rc=0 message thinking temperature [[ -n "${KIMI_API_KEY:-}" ]] || die "kimi seat requires KIMI_API_KEY" command -v curl >/dev/null 2>&1 || die "kimi seat requires curl" command -v jq >/dev/null 2>&1 || die "kimi seat requires jq" + # kimi-k3 reasons by default and Moonshot bills reasoning against max_tokens, + # so a capped caller gets finish_reason="length" with empty content. Setting + # KIMI_THINKING=disabled selects the non-thinking mode, which the API accepts + # only at temperature 0.6 (thinking mode only accepts 1). Unset is the + # provider default and leaves the document seats byte-identical. + thinking="${KIMI_THINKING:-}" + case "$thinking" in + "") temperature="${KIMI_TEMPERATURE:-1}" ;; + disabled) temperature="${KIMI_TEMPERATURE:-0.6}" ;; + *) die "kimi seat: KIMI_THINKING must be empty or disabled" ;; + esac + [[ "$temperature" =~ ^[0-9]+(\.[0-9]+)?$ ]] || die "kimi seat: KIMI_TEMPERATURE must be a number" request_file=$(mktemp -t kimi-request-XXXXXX.json) response_file=$(mktemp -t kimi-response-XXXXXX.json) @@ -678,12 +704,16 @@ invoke_kimi() ( # The document seat calls Kimi's model API directly. Kimi Code's -p agent # loop auto-runs Read/Write tools, which violates the no-tools seat boundary. - jq -Rs --arg model "${KIMI_MODEL:-kimi-k3}" '{ + jq -Rs --arg model "${KIMI_MODEL:-kimi-k3}" --argjson max_tokens "${KIMI_MAX_TOKENS:-0}" \ + --argjson temperature "$temperature" --arg thinking "$thinking" '{ model: $model, messages: [{role: "user", content: .}], stream: false, - temperature: 1 - }' "$prompt_file" > "$request_file" + temperature: $temperature + } + + (if $max_tokens > 0 then {max_tokens: $max_tokens} else {} end) + + (if $thinking != "" then {thinking: {type: $thinking}} else {} end)' \ + "$prompt_file" > "$request_file" { printf 'url = "https://api.moonshot.ai/v1/chat/completions"\n' diff --git a/tests/kimi-seat-simulation.sh b/tests/kimi-seat-simulation.sh index 787be1d..20983af 100755 --- a/tests/kimi-seat-simulation.sh +++ b/tests/kimi-seat-simulation.sh @@ -9,6 +9,12 @@ BOUNCER="$REPO_ROOT/co-evolve-bouncer.sh" TEST_DIR="$(mktemp -d -t kimi-seat-XXXXXX)" trap 'rm -rf "$TEST_DIR"' EXIT +# The bouncer also reads seat keys from the repo's .env.local, which on a +# developer machine holds a real KIMI_API_KEY and would defeat the missing-key +# scenario below. Point every scenario at a path that does not exist so the +# fixture, not the machine, decides which keys are present. +export CO_EVOLVE_ENV_FILE="$TEST_DIR/absent.env" + TOTAL=0 FAILURES=0 pass() { printf 'PASS: %s\n' "$1"; } diff --git a/tests/run-all.sh b/tests/run-all.sh index 4c2f2a0..564655b 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -139,6 +139,7 @@ if [[ "$QUICK" == false ]]; then SUITES+=("$REPO_ROOT/benchmarks/tests/test-panel.sh") SUITES+=("$REPO_ROOT/benchmarks/tests/test-judging.sh") SUITES+=("$REPO_ROOT/benchmarks/tests/test-report.sh") + SUITES+=("$REPO_ROOT/benchmarks/code/tests/test-code-bench.sh") fi # --- run -----------------------------------------------------------------------