From 124728c4a9c6f6645ef9c676ecbeb3d8e98ff6ae Mon Sep 17 00:00:00 2001 From: Alan Shurafa Date: Mon, 31 Aug 2026 01:18:11 -0400 Subject: [PATCH 01/10] Add compute-capped code benchmark battery Co-Authored-By: Codex --- .../2026-08-31-code-benchmark-battery-plan.md | 57 +++++ benchmarks/README.md | 5 + benchmarks/code/README.md | 105 +++++++++ benchmarks/code/code-bench.sh | 68 ++++++ benchmarks/code/conditions.json | 29 +++ benchmarks/code/drivers/run-workflow.sh | 206 ++++++++++++++++++ benchmarks/code/estimate-compute.sh | 83 +++++++ benchmarks/code/external-sources.lock.json | 12 + benchmarks/code/lib/code-bench-lib.sh | 82 +++++++ .../code/schemas/prediction.schema.json | 12 + benchmarks/code/scripts/evaluate-swebench.sh | 42 ++++ .../code/scripts/fetch-swebench-metadata.py | 80 +++++++ .../code/scripts/prepare-swebench-instance.sh | 48 ++++ benchmarks/code/scripts/run-canary.sh | 64 ++++++ benchmarks/code/scripts/setup-swebench.sh | 67 ++++++ .../subsets/swebench-verified-canary.json | 12 + benchmarks/code/suites.json | 13 ++ benchmarks/code/tests/test-code-bench.sh | 123 +++++++++++ benchmarks/code/validate-predictions.sh | 41 ++++ tests/run-all.sh | 1 + 20 files changed, 1150 insertions(+) create mode 100644 .planning/notes/2026-08-31-code-benchmark-battery-plan.md create mode 100644 benchmarks/code/README.md create mode 100644 benchmarks/code/code-bench.sh create mode 100644 benchmarks/code/conditions.json create mode 100644 benchmarks/code/drivers/run-workflow.sh create mode 100644 benchmarks/code/estimate-compute.sh create mode 100644 benchmarks/code/external-sources.lock.json create mode 100644 benchmarks/code/lib/code-bench-lib.sh create mode 100644 benchmarks/code/schemas/prediction.schema.json create mode 100644 benchmarks/code/scripts/evaluate-swebench.sh create mode 100644 benchmarks/code/scripts/fetch-swebench-metadata.py create mode 100644 benchmarks/code/scripts/prepare-swebench-instance.sh create mode 100644 benchmarks/code/scripts/run-canary.sh create mode 100644 benchmarks/code/scripts/setup-swebench.sh create mode 100644 benchmarks/code/subsets/swebench-verified-canary.json create mode 100644 benchmarks/code/suites.json create mode 100644 benchmarks/code/tests/test-code-bench.sh create mode 100644 benchmarks/code/validate-predictions.sh diff --git a/.planning/notes/2026-08-31-code-benchmark-battery-plan.md b/.planning/notes/2026-08-31-code-benchmark-battery-plan.md new file mode 100644 index 0000000..4150187 --- /dev/null +++ b/.planning/notes/2026-08-31-code-benchmark-battery-plan.md @@ -0,0 +1,57 @@ +# Code Benchmark Battery — Implementation Plan + +Date: 2026-08-31 + +## Goal + +Measure whether Co-Evolution workflows produce repository patches that pass +standard deterministic evaluators, while keeping subscription usage bounded and +making every live dispatch auditable before it runs. + +## Conditions + +- **A — Fable solo:** one Fable coding-agent dispatch. +- **B — cross-vendor bounce:** Fable implements; Codex reviews and repairs. +- **C — Fable-led panel:** Fable implements; Codex, GLM, and Kimi critique; + Fable performs the final repair. +- **D — Fable self-bounce:** Fable implements and then reviews/repairs its own + patch. This controls for extra passes and compute. + +Declared provider dispatches are a lower bound: one coding-agent dispatch may +contain multiple internal model turns. The live runner must enforce a declared +Claude-dispatch cap and must never infer a percentage of a Max subscription from +tokens because Anthropic does not publish a fixed weekly token denominator. + +## Delivery sequence + +1. Add frozen suite, condition, external-source, and subset manifests. +2. Add an offline compute estimator and fail-closed prediction validator. +3. Add a pinned SWE-bench installer, metadata fetcher, and official evaluator + bridge under an ignored cache/results tree. +4. Add hermetic tests and wire them into the repository aggregate gate. +5. Prepare the five-task canary and validate official gold scoring when Docker + is available. +6. Add live patch-generation drivers only after the offline harness is green. + Begin with one task and conditions A/B/C, capped at four declared Fable + dispatches. Measure the actual Settings > Usage change before expanding. + +## Safety and scope + +- No full SWE-bench run in this phase. +- No live model calls during setup or hermetic verification. +- No gold patches, hidden tests, or oracle solutions are exposed to agents. +- Every condition starts from the same clean instance and produces a standard + prediction record for the official evaluator. +- Raw datasets, images, virtual environments, trajectories, and results stay + below `benchmarks/results/code/` and remain uncommitted. + +## Acceptance criteria + +- `bash benchmarks/code/code-bench.sh check` passes. +- The estimator reports exact declared dispatch counts and refuses a cap breach + with exit 75. +- Prediction validation rejects unknown instances, duplicates, empty patches, + and malformed JSONL. +- The pinned metadata fetch contains only public task inputs, never gold data. +- The official SWE-bench gold canary passes once Docker is running. +- The repository aggregate test gate includes the new hermetic suite. diff --git a/benchmarks/README.md b/benchmarks/README.md index 487d3e8..9c760ce 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,5 +1,10 @@ # Co-Evolution Benchmark Suite +Repository-patch and coding-agent evaluation lives in +[`benchmarks/code/`](code/README.md). It uses official deterministic evaluators +and a separate compute-cap contract; this document continues to describe the +plan-composition benchmark. + Batch runbook for comparing plan-composition conditions (solo Fable, Codex bounce, panel critique, self-bounce control) on identical planning tasks, scored by three blind automated judges. See `PREREGISTRATION.md` for the diff --git a/benchmarks/code/README.md b/benchmarks/code/README.md new file mode 100644 index 0000000..9395a94 --- /dev/null +++ b/benchmarks/code/README.md @@ -0,0 +1,105 @@ +# Code Benchmark Battery + +This suite compares Co-Evolution coding workflows using deterministic external +evaluators. It complements the plan-composition benchmark in `benchmarks/`; +it does not reuse model judges when official tests can decide whether a patch +resolved an issue. + +Authoritative references: [SWE-bench quickstart](https://www.swebench.com/SWE-bench/guides/quickstart/), +[SWE-bench Verified limitations](https://openai.com/index/introducing-swe-bench-verified/), +[Aider benchmark harness](https://github.com/Aider-AI/aider/blob/main/benchmark/README.md), +and [Harbor Terminal-Bench](https://www.harborframework.com/docs/tutorials/running-terminal-bench). + +## Initial battery + +- **SWE-bench Verified canary:** five pinned tasks from five repositories. +- **Local fixture lane:** hermetic contracts for the runner itself; no models. +- **Future lanes:** Aider Polyglot, Terminal-Bench, and a private recent-issue + set can plug into the same prediction and reporting contract. + +The four conditions are declared in `conditions.json`. Condition D is retained +as a self-bounce control even when the product question focuses on A/B/C. + +## Zero-compute setup + +```bash +bash benchmarks/code/code-bench.sh check +bash benchmarks/code/code-bench.sh estimate --suite swebench-verified-canary +bash benchmarks/code/code-bench.sh fetch-metadata +bash benchmarks/code/code-bench.sh setup --install +``` + +These commands make no model calls. The installer pins both the official +SWE-bench repository and dataset revisions from `external-sources.lock.json`. +External files live under the ignored `benchmarks/results/code/` tree. + +## Capped patch generation + +Prepare a clean workspace, then run a single condition. Preparation makes no +model calls. `run-workflow` refuses to start without an explicit Claude cap. + +```bash +input=$(bash benchmarks/code/code-bench.sh prepare-instance \ + sympy__sympy-20916 calibration-1 A) + +bash benchmarks/code/code-bench.sh run-workflow \ + --input "$input" \ + --predictions benchmarks/results/code/predictions/calibration-1.jsonl \ + --max-claude-dispatches 1 +``` + +Use `--dry-run` on `run-workflow` to inspect its phase plan without invoking a +provider. Condition C labels its three critiques anonymously and gives Fable +the final repair decision. If `ANTHROPIC_API_KEY` is present, live generation +fails closed so Claude Console credits cannot be charged accidentally instead +of the Max subscription. + +Live phases default to medium reasoning and a 900-second timeout. Override with +`CODE_BENCH_CLAUDE_EFFORT`, `CODE_BENCH_CODEX_EFFORT`, and +`CODE_BENCH_PHASE_TIMEOUT`; changing these values creates a different treatment +and must be recorded in the run manifest. + +## Official evaluator + +Docker must be running. Validate the environment with a gold patch before +evaluating generated predictions: + +```bash +bash benchmarks/code/code-bench.sh gold-canary +bash benchmarks/code/code-bench.sh validate-predictions predictions.jsonl +bash benchmarks/code/code-bench.sh evaluate predictions.jsonl +``` + +Prediction JSONL uses the official SWE-bench fields: + +```json +{"instance_id":"owner__repo-123","model_name_or_path":"condition-A","model_patch":"diff --git ..."} +``` + +Gold patches and hidden tests are never stored in this repository or supplied +to a generation workflow. + +## Compute contract + +`estimate` reports declared provider dispatches. For the five-task canary: + +- A/B/C: 20 declared Fable dispatches total. +- A/B/C/D: 30 declared Fable dispatches total. +- One task across A/B/C: 4 declared Fable dispatches. + +Those are lower bounds because a coding-agent session may contain multiple +model turns. Live generation must begin with one task, A/B/C, and a cap of four +declared Fable dispatches. Anthropic does not publish a fixed weekly token +allowance, so the first run is also the calibration: record Settings > Usage +before and after, then use that measured delta to decide whether to expand. + +The batch command enforces that aggregate cap before cloning or dispatching: + +```bash +bash benchmarks/code/code-bench.sh run-canary \ + --run-id calibration-1 \ + --conditions A,B,C \ + --task-limit 1 \ + --max-claude-dispatches 4 \ + --dry-run +``` diff --git a/benchmarks/code/code-bench.sh b/benchmarks/code/code-bench.sh new file mode 100644 index 0000000..931402b --- /dev/null +++ b/benchmarks/code/code-bench.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/code-bench-lib.sh +source "$SCRIPT_DIR/lib/code-bench-lib.sh" + +COMMAND="${1:-}" +shift || true + +case "$COMMAND" in + check) + code_check_manifests + ;; + estimate) + exec bash "$SCRIPT_DIR/estimate-compute.sh" "$@" + ;; + fetch-metadata) + suite_json=$(code_suite_json "swebench-verified-canary") + subset=$(code_subset_path "$suite_json") + output="$CODE_BENCH_RESULTS_ROOT/metadata/swebench-verified-canary.json" + exec python "$SCRIPT_DIR/scripts/fetch-swebench-metadata.py" \ + --subset "$subset" --lock "$SCRIPT_DIR/external-sources.lock.json" --output "$output" + ;; + setup) + exec bash "$SCRIPT_DIR/scripts/setup-swebench.sh" "$@" + ;; + prepare-instance) + exec bash "$SCRIPT_DIR/scripts/prepare-swebench-instance.sh" "$@" + ;; + run-workflow) + exec bash "$SCRIPT_DIR/drivers/run-workflow.sh" "$@" + ;; + run-canary) + exec bash "$SCRIPT_DIR/scripts/run-canary.sh" "$@" + ;; + validate-predictions) + exec bash "$SCRIPT_DIR/validate-predictions.sh" "$@" + ;; + gold-canary) + exec bash "$SCRIPT_DIR/scripts/evaluate-swebench.sh" gold "$@" + ;; + evaluate) + exec bash "$SCRIPT_DIR/scripts/evaluate-swebench.sh" predictions "$@" + ;; + -h|--help|help|"") + cat <<'USAGE' +usage: code-bench.sh COMMAND [options] + + check validate checked-in manifests + estimate [options] report declared provider dispatches + fetch-metadata cache public inputs for the frozen subset + setup --check|--install inspect or install pinned SWE-bench tooling + prepare-instance ID RUN COND clone a clean public-input workspace + run-workflow [options] generate one capped condition prediction + run-canary [options] run a batch with one aggregate Claude cap + validate-predictions FILE validate JSONL before official scoring + gold-canary [INSTANCE] verify the official evaluator with a gold patch + evaluate FILE score generated predictions officially + +No command in this phase invokes a model. Live generation drivers are added +only after this offline harness and its compute cap are verified. +USAGE + ;; + *) + code_die "unknown command: $COMMAND"; exit 2 + ;; +esac diff --git a/benchmarks/code/conditions.json b/benchmarks/code/conditions.json new file mode 100644 index 0000000..d1ce7be --- /dev/null +++ b/benchmarks/code/conditions.json @@ -0,0 +1,29 @@ +{ + "schema": "code-bench-conditions/1.0", + "conditions": [ + { + "id": "A", + "label": "fable-solo", + "description": "Fable investigates, edits, and tests once.", + "dispatches": {"claude": 1, "codex": 0, "glm": 0, "kimi": 0} + }, + { + "id": "B", + "label": "cross-vendor-bounce", + "description": "Fable implements; Codex reviews and repairs the patch.", + "dispatches": {"claude": 1, "codex": 1, "glm": 0, "kimi": 0} + }, + { + "id": "C", + "label": "fable-led-panel", + "description": "Fable implements; Codex, GLM, and Kimi critique; Fable repairs.", + "dispatches": {"claude": 2, "codex": 1, "glm": 1, "kimi": 1} + }, + { + "id": "D", + "label": "fable-self-bounce", + "description": "Fable implements and performs a second review/repair pass.", + "dispatches": {"claude": 2, "codex": 0, "glm": 0, "kimi": 0} + } + ] +} diff --git a/benchmarks/code/drivers/run-workflow.sh b/benchmarks/code/drivers/run-workflow.sh new file mode 100644 index 0000000..e697b5a --- /dev/null +++ b/benchmarks/code/drivers/run-workflow.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../lib/code-bench-lib.sh +source "$CODE_DIR/lib/code-bench-lib.sh" +# shellcheck source=../../../lib/co-evolution.sh +source "$CODE_BENCH_REPO_ROOT/lib/co-evolution.sh" + +INPUT_JSON="" +PREDICTIONS="" +MAX_CLAUDE="" +DRY_RUN=false +CLAUDE_MODEL="${CODE_BENCH_CLAUDE_MODEL:-fable}" +CODEX_MODEL_LOCAL="${CODE_BENCH_CODEX_MODEL:-gpt-5.6-sol}" +CLAUDE_EFFORT_LOCAL="${CODE_BENCH_CLAUDE_EFFORT:-medium}" +CODEX_EFFORT_LOCAL="${CODE_BENCH_CODEX_EFFORT:-medium}" +PHASE_TIMEOUT="${CODE_BENCH_PHASE_TIMEOUT:-900}" + +while (( $# > 0 )); do + case "$1" in + --input) INPUT_JSON="${2:?--input needs a value}"; shift 2 ;; + --predictions) PREDICTIONS="${2:?--predictions needs a value}"; shift 2 ;; + --max-claude-dispatches) MAX_CLAUDE="${2:?--max-claude-dispatches needs a value}"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + *) code_die "unknown workflow option: $1"; exit 2 ;; + esac +done + +[[ -f "$INPUT_JSON" ]] || { code_die "--input must name a prepared input.json"; exit 2; } +[[ -n "$PREDICTIONS" ]] || { code_die "--predictions is required"; exit 2; } +[[ "$MAX_CLAUDE" =~ ^[0-9]+$ ]] || { code_die "--max-claude-dispatches is required and must be an integer"; exit 2; } +[[ "$PHASE_TIMEOUT" =~ ^[1-9][0-9]*$ ]] || { code_die "CODE_BENCH_PHASE_TIMEOUT must be positive"; exit 2; } + +instance=$(jq -r '.instance_id' "$INPUT_JSON" | tr -d '\r') +condition=$(jq -r '.condition' "$INPUT_JSON" | tr -d '\r') +workspace=$(jq -r '.workspace' "$INPUT_JSON" | tr -d '\r') +task_file=$(jq -r '.task_file' "$INPUT_JSON" | tr -d '\r') +[[ -d "$workspace/.git" && -f "$task_file" ]] || { code_die "prepared workspace or task file is missing"; exit 1; } + +results_root=$(cd "$CODE_BENCH_RESULTS_ROOT" && pwd -P) +input_dir=$(cd "$(dirname "$INPUT_JSON")" && pwd -P) +input_abs="$input_dir/$(basename "$INPUT_JSON")" +workspace_abs=$(cd "$workspace" && pwd -P) +task_dir=$(cd "$(dirname "$task_file")" && pwd -P) +task_abs="$task_dir/$(basename "$task_file")" +case "$input_abs" in "$results_root"/runs/*/*/*/input.json) ;; *) code_die "input.json is outside the benchmark run sandbox"; exit 1 ;; esac +[[ "$workspace_abs" == "$input_dir/workspace" ]] || { code_die "workspace does not belong to the prepared cell"; exit 1; } +[[ "$task_abs" == "$input_dir/task.md" ]] || { code_die "task file does not belong to the prepared cell"; exit 1; } +mkdir -p "$(dirname "$PREDICTIONS")" +pred_dir=$(cd "$(dirname "$PREDICTIONS")" && pwd -P) +pred_abs="$pred_dir/$(basename "$PREDICTIONS")" +case "$pred_abs" in "$results_root"/predictions/*) ;; *) code_die "predictions path is outside the benchmark prediction sandbox"; exit 1 ;; esac + +condition_json=$(jq -ce --arg id "$condition" '.conditions | map(select(.id == $id)) | if length == 1 then .[0] else empty end' "$CODE_DIR/conditions.json") \ + || { code_die "unknown condition: $condition"; exit 1; } +claude_needed=$(printf '%s' "$condition_json" | jq -r '.dispatches.claude' | tr -d '\r') +if (( claude_needed > MAX_CLAUDE )); then + printf 'REFUSED: condition %s declares %s Claude dispatches; cap is %s.\n' "$condition" "$claude_needed" "$MAX_CLAUDE" >&2 + exit 75 +fi + +case "$condition" in + A) phases="fable-implement" ;; + B) phases="fable-implement,codex-repair" ;; + C) phases="fable-implement,codex-critique,glm-critique,kimi-critique,fable-repair" ;; + D) phases="fable-implement,fable-self-repair" ;; +esac +if [[ "$DRY_RUN" == true ]]; then + jq -n --arg instance "$instance" --arg condition "$condition" --arg phases "$phases" \ + --argjson claude "$claude_needed" \ + '{instance:$instance,condition:$condition,phases:($phases|split(",")),declared_claude_dispatches:$claude,executed:false}' + exit 0 +fi + +if [[ -n "${ANTHROPIC_API_KEY:-}" ]]; then + code_die "ANTHROPIC_API_KEY is set; refusing because this would bill API credits instead of the Max plan" + exit 1 +fi +command -v claude >/dev/null 2>&1 || { code_die "claude CLI is required"; exit 1; } +command -v codex >/dev/null 2>&1 || { code_die "codex CLI is required"; exit 1; } + +load_named_key() { + local name="$1" env_file="$CODE_BENCH_REPO_ROOT/.env.local" line="" value="" + [[ -z "${!name:-}" && -r "$env_file" ]] || return 0 + line=$(grep -m 1 -E "^[[:space:]]*(export[[:space:]]+)?${name}[[:space:]]*=" "$env_file" 2>/dev/null || true) + [[ -n "$line" ]] || return 0 + value=$(printf '%s' "$line" | sed -e 's/^[^=]*=//' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + case "$value" in + \"*\") value="${value#\"}"; value="${value%\"}" ;; + \'*\') value="${value#\'}"; value="${value%\'}" ;; + esac + [[ -n "$value" ]] && printf -v "$name" '%s' "$value" +} +load_named_key ZAI_API_KEY +load_named_key KIMI_API_KEY + +cell="$input_dir" +logs="$cell/logs" +reviews="$cell/reviews" +mkdir -p "$logs" "$reviews" +jq -n --arg instance "$instance" --arg condition "$condition" \ + --arg claude_model "$CLAUDE_MODEL" --arg claude_effort "$CLAUDE_EFFORT_LOCAL" \ + --arg codex_model "$CODEX_MODEL_LOCAL" --arg codex_effort "$CODEX_EFFORT_LOCAL" \ + --argjson phase_timeout "$PHASE_TIMEOUT" --argjson declared_claude "$claude_needed" \ + '{schema:"code-bench-run/1.0",instance:$instance,condition:$condition, + models:{claude:$claude_model,codex:$codex_model}, + effort:{claude:$claude_effort,codex:$codex_effort}, + phase_timeout_seconds:$phase_timeout,declared_claude_dispatches:$declared_claude}' \ + > "$cell/run-manifest.json" + +write_implement_prompt() { + local out="$1" + { + printf '%s\n\n' "You are solving a repository issue in the current working directory." + printf '%s\n' "Inspect the code, implement the smallest correct fix, and run the most relevant tests available." + printf '%s\n' "Do not read outside the repository. Do not commit. Do not modify tests merely to make a failure disappear." + printf '\n## ISSUE\n\n' + cat "$task_file" + } > "$out" +} + +run_fable() { + local phase="$1" prompt="$2" + local -a cmd=(claude -p --model "$CLAUDE_MODEL" --effort "$CLAUDE_EFFORT_LOCAL" + --safe-mode --permission-mode auto --tools "Bash,Read,Edit,Write,Glob,Grep" + --no-session-persistence --output-format json) + command -v timeout >/dev/null 2>&1 && cmd=(timeout --foreground "${PHASE_TIMEOUT}s" "${cmd[@]}") + (cd "$workspace" && "${cmd[@]}" < "$prompt") \ + > "$logs/$phase.json" 2> "$logs/$phase.stderr.log" +} + +run_codex_repair() { + local prompt="$1" + local -a cmd=(codex exec -C "$workspace" -m "$CODEX_MODEL_LOCAL" --sandbox workspace-write + --ephemeral --ignore-user-config -c approval_policy="never" + -c model_reasoning_effort="$CODEX_EFFORT_LOCAL" -) + command -v timeout >/dev/null 2>&1 && cmd=(timeout --foreground "${PHASE_TIMEOUT}s" "${cmd[@]}") + "${cmd[@]}" \ + < "$prompt" > "$logs/codex-repair.log" 2> "$logs/codex-repair.stderr.log" +} + +run_codex_critique() { + local prompt="$1" out="$2" + local -a cmd=(codex exec -C "$workspace" -m "$CODEX_MODEL_LOCAL" --sandbox read-only + --ephemeral --ignore-user-config -c approval_policy="never" + -c model_reasoning_effort="$CODEX_EFFORT_LOCAL" -o "$out" -) + command -v timeout >/dev/null 2>&1 && cmd=(timeout --foreground "${PHASE_TIMEOUT}s" "${cmd[@]}") + "${cmd[@]}" < "$prompt" > "$logs/codex-critique.log" 2> "$logs/codex-critique.stderr.log" +} + +write_implement_prompt "$cell/implement-prompt.md" +run_fable fable-implement "$cell/implement-prompt.md" + +case "$condition" in + B) + { + printf '%s\n' "Review the current uncommitted implementation for the issue below. Inspect the diff and repository, correct defects, and run relevant tests. Do not commit." + printf '\n## ISSUE\n\n'; cat "$task_file" + } > "$cell/codex-repair-prompt.md" + run_codex_repair "$cell/codex-repair-prompt.md" + ;; + C) + git -C "$workspace" diff --binary > "$cell/candidate.patch" + { + printf '%s\n' "Critique the candidate patch for correctness, regressions, missing cases, and scope. Do not edit files. Return concrete findings only." + printf '\n## ISSUE\n\n'; cat "$task_file" + printf '\n## CANDIDATE PATCH\n\n'; head -c 120000 "$cell/candidate.patch" + } > "$cell/critique-prompt.md" + run_codex_critique "$cell/critique-prompt.md" "$reviews/reviewer-1.md" + [[ -n "${ZAI_API_KEY:-}" ]] || { code_die "condition C requires ZAI_API_KEY"; exit 1; } + [[ -n "${KIMI_API_KEY:-}" ]] || { code_die "condition C requires KIMI_API_KEY"; exit 1; } + invoke_glm "$cell/critique-prompt.md" "$reviews/reviewer-2.md" "$logs/glm-critique.stderr.log" false + invoke_kimi "$cell/critique-prompt.md" "$reviews/reviewer-3.md" "$logs/kimi-critique.stderr.log" + { + printf '%s\n' "Re-open the current implementation and evaluate the three anonymous reviews below. Decide every finding on its merits, repair accepted issues, and run relevant tests. Do not commit." + printf '\n## ISSUE\n\n'; cat "$task_file" + reviewer_number=0 + for review in "$reviews/reviewer-1.md" "$reviews/reviewer-2.md" "$reviews/reviewer-3.md"; do + reviewer_number=$((reviewer_number + 1)) + printf '\n## REVIEWER %s\n\n' "$reviewer_number" + head -c 40000 "$review" + printf '\n' + done + } > "$cell/fable-repair-prompt.md" + run_fable fable-repair "$cell/fable-repair-prompt.md" + ;; + D) + { + printf '%s\n' "Review your current uncommitted implementation for the issue below. Find and repair correctness or regression risks and run relevant tests. Do not commit." + printf '\n## ISSUE\n\n'; cat "$task_file" + } > "$cell/fable-self-repair-prompt.md" + run_fable fable-self-repair "$cell/fable-self-repair-prompt.md" + ;; +esac + +patch="$cell/final.patch" +git -C "$workspace" diff --binary > "$patch" +[[ -s "$patch" ]] || { code_die "workflow produced an empty patch"; exit 1; } +record="$cell/prediction.json" +jq -n --arg instance_id "$instance" --arg model "co-evolution-condition-$condition" \ + --rawfile model_patch "$patch" \ + '{instance_id:$instance_id,model_name_or_path:$model,model_patch:$model_patch}' > "$record" +jq -c . "$record" >> "$PREDICTIONS" +printf 'WROTE: %s\n' "$record" diff --git a/benchmarks/code/estimate-compute.sh b/benchmarks/code/estimate-compute.sh new file mode 100644 index 0000000..bcec8fa --- /dev/null +++ b/benchmarks/code/estimate-compute.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/code-bench-lib.sh +source "$SCRIPT_DIR/lib/code-bench-lib.sh" + +SUITE="swebench-verified-canary" +CONDITIONS="" +MAX_CLAUDE="" +TASK_LIMIT="" +JSON=false + +while (( $# > 0 )); do + case "$1" in + --suite) SUITE="${2:?--suite needs a value}"; shift 2 ;; + --conditions) CONDITIONS="${2:?--conditions needs a value}"; shift 2 ;; + --max-claude-dispatches) MAX_CLAUDE="${2:?--max-claude-dispatches needs a value}"; shift 2 ;; + --task-limit) TASK_LIMIT="${2:?--task-limit needs a value}"; shift 2 ;; + --json) JSON=true; shift ;; + *) code_die "unknown estimate option: $1"; exit 2 ;; + esac +done + +suite_json=$(code_suite_json "$SUITE") || { code_die "unknown suite: $SUITE"; exit 1; } +subset=$(code_subset_path "$suite_json") +tasks=$(jq '.instances | length' "$subset" | tr -d '\r') +if [[ -n "$TASK_LIMIT" ]]; then + [[ "$TASK_LIMIT" =~ ^[1-9][0-9]*$ ]] || { code_die "--task-limit must be a positive integer"; exit 2; } + (( TASK_LIMIT <= tasks )) || { code_die "--task-limit $TASK_LIMIT exceeds suite size $tasks"; exit 2; } + tasks="$TASK_LIMIT" +fi + +if [[ -z "$CONDITIONS" ]]; then + CONDITIONS=$(printf '%s' "$suite_json" | jq -r '.default_conditions | join(",")' | tr -d '\r') +fi + +selected='[]' +old_ifs=$IFS +IFS=',' +for condition in $CONDITIONS; do + row=$(jq -ce --arg id "$condition" '.conditions | map(select(.id == $id)) | if length == 1 then .[0] else empty end' "$SCRIPT_DIR/conditions.json") \ + || { IFS=$old_ifs; code_die "unknown condition: $condition"; exit 1; } + selected=$(jq -c --argjson row "$row" '. + [$row]' <<<"$selected") +done +IFS=$old_ifs + +summary=$(jq -cn --arg suite "$SUITE" --argjson tasks "$tasks" --argjson conditions "$selected" ' + def total($p): ([$conditions[].dispatches[$p]] | add // 0) * $tasks; + { + schema: "code-bench-compute-estimate/1.0", + suite: $suite, + tasks: $tasks, + conditions: [$conditions[].id], + cells: ($tasks * ($conditions | length)), + declared_dispatches: { + claude: total("claude"), codex: total("codex"), + glm: total("glm"), kimi: total("kimi") + }, + caveat: "Declared dispatches are a lower bound; a coding-agent dispatch may contain multiple model turns." + }') + +claude_calls=$(printf '%s' "$summary" | jq -r '.declared_dispatches.claude' | tr -d '\r') +if [[ -n "$MAX_CLAUDE" ]]; then + [[ "$MAX_CLAUDE" =~ ^[0-9]+$ ]] || { code_die "--max-claude-dispatches must be a non-negative integer"; exit 2; } + if (( claude_calls > MAX_CLAUDE )); then + printf 'REFUSED: estimate requires %s declared Claude dispatches; cap is %s.\n' "$claude_calls" "$MAX_CLAUDE" >&2 + exit 75 + fi +fi + +if [[ "$JSON" == true ]]; then + printf '%s\n' "$summary" | jq . +else + printf 'Compute estimate — %s\n' "$SUITE" + printf ' tasks: %s | conditions: %s | cells: %s\n' \ + "$tasks" "$(printf '%s' "$summary" | jq -r '.conditions | join(",")')" "$(printf '%s' "$summary" | jq -r '.cells')" + for provider in claude codex glm kimi; do + printf ' %-6s declared dispatches: %s\n' "$provider" "$(printf '%s' "$summary" | jq -r --arg p "$provider" '.declared_dispatches[$p]')" + done + printf ' NOTE: one coding-agent dispatch may contain multiple internal model turns.\n' + printf ' Weekly-Max percentage: not derivable without the account usage meter; calibrate with one capped task.\n' +fi diff --git a/benchmarks/code/external-sources.lock.json b/benchmarks/code/external-sources.lock.json new file mode 100644 index 0000000..2a437d9 --- /dev/null +++ b/benchmarks/code/external-sources.lock.json @@ -0,0 +1,12 @@ +{ + "schema": "code-bench-external-lock/1.0", + "swebench": { + "repository": "https://github.com/SWE-bench/SWE-bench.git", + "commit": "7a21e05772954cc81471ae19d56f436cecf43c54" + }, + "dataset": { + "id": "princeton-nlp/SWE-bench_Verified", + "revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a", + "split": "test" + } +} diff --git a/benchmarks/code/lib/code-bench-lib.sh b/benchmarks/code/lib/code-bench-lib.sh new file mode 100644 index 0000000..406d320 --- /dev/null +++ b/benchmarks/code/lib/code-bench-lib.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +CODE_BENCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CODE_BENCH_REPO_ROOT="$(cd "$CODE_BENCH_DIR/../.." && pwd)" +CODE_BENCH_RESULTS_ROOT="${CODE_BENCH_RESULTS_ROOT:-$CODE_BENCH_REPO_ROOT/benchmarks/results/code}" + +code_die() { + printf 'ERROR: %s\n' "$*" >&2 + return 1 +} + +code_require() { + command -v "$1" >/dev/null 2>&1 || code_die "$1 is required" +} + +code_suite_json() { + local suite="$1" + jq -ce --arg id "$suite" '.suites[] | select(.id == $id)' "$CODE_BENCH_DIR/suites.json" +} + +code_subset_path() { + local suite_json="$1" rel + rel=$(printf '%s' "$suite_json" | jq -r '.subset_file' | tr -d '\r') + printf '%s/%s' "$CODE_BENCH_DIR" "$rel" +} + +code_check_manifests() { + local failures=0 suite_json subset count declared + code_require jq || return 1 + + jq -e '.schema == "code-bench-conditions/1.0" and + (.conditions | length > 0) and + ([.conditions[].id] | length == (unique | length)) and + (all(.conditions[]; (.dispatches | keys) == ["claude","codex","glm","kimi"])) and + ([.conditions[].dispatches[] | type == "number" and . >= 0 and floor == .] | all)' \ + "$CODE_BENCH_DIR/conditions.json" >/dev/null || { + printf 'CHECK FAIL: conditions.json\n' >&2; failures=$((failures + 1)); + } + jq -e '.schema == "code-bench-suites/1.0" and + (.suites | length > 0) and + ([.suites[].id] | length == (unique | length))' \ + "$CODE_BENCH_DIR/suites.json" >/dev/null || { + printf 'CHECK FAIL: suites.json\n' >&2; failures=$((failures + 1)); + } + jq -e '.schema == "code-bench-external-lock/1.0" and + (.swebench.commit | test("^[0-9a-f]{40}$")) and + (.dataset.revision | test("^[0-9a-f]{40}$"))' \ + "$CODE_BENCH_DIR/external-sources.lock.json" >/dev/null || { + printf 'CHECK FAIL: external-sources.lock.json\n' >&2; failures=$((failures + 1)); + } + + while IFS= read -r suite_json; do + subset=$(code_subset_path "$suite_json") + declared=$(printf '%s' "$suite_json" | jq -r '.task_count' | tr -d '\r') + if [[ ! -f "$subset" ]]; then + printf 'CHECK FAIL: subset missing: %s\n' "$subset" >&2 + failures=$((failures + 1)); continue + fi + if ! jq -e '.schema == "code-bench-subset/1.0" and + (.instances | length > 0) and + ([.instances[].instance_id] | length == (unique | length)) and + ([.instances[].repo] | length == (unique | length))' "$subset" >/dev/null; then + printf 'CHECK FAIL: malformed or duplicate subset entries: %s\n' "$subset" >&2 + failures=$((failures + 1)); continue + fi + count=$(jq '.instances | length' "$subset" | tr -d '\r') + if [[ "$count" != "$declared" ]]; then + printf 'CHECK FAIL: suite declares %s tasks but subset contains %s: %s\n' "$declared" "$count" "$subset" >&2 + failures=$((failures + 1)) + fi + while IFS= read -r condition; do + jq -e --arg id "$condition" 'any(.conditions[]; .id == $id)' \ + "$CODE_BENCH_DIR/conditions.json" >/dev/null || { + printf 'CHECK FAIL: suite references unknown condition %s\n' "$condition" >&2 + failures=$((failures + 1)) + } + done < <(printf '%s' "$suite_json" | jq -r '.default_conditions[]' | tr -d '\r') + done < <(jq -c '.suites[]' "$CODE_BENCH_DIR/suites.json") + + (( failures == 0 )) || return 1 + printf 'CHECK: code benchmark manifests PASS\n' +} diff --git a/benchmarks/code/schemas/prediction.schema.json b/benchmarks/code/schemas/prediction.schema.json new file mode 100644 index 0000000..89cd943 --- /dev/null +++ b/benchmarks/code/schemas/prediction.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "SWE-bench prediction record", + "type": "object", + "additionalProperties": true, + "required": ["instance_id", "model_name_or_path", "model_patch"], + "properties": { + "instance_id": {"type": "string", "minLength": 1}, + "model_name_or_path": {"type": "string", "minLength": 1}, + "model_patch": {"type": "string", "minLength": 1} + } +} diff --git a/benchmarks/code/scripts/evaluate-swebench.sh b/benchmarks/code/scripts/evaluate-swebench.sh new file mode 100644 index 0000000..1a4d1de --- /dev/null +++ b/benchmarks/code/scripts/evaluate-swebench.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../lib/code-bench-lib.sh +source "$CODE_DIR/lib/code-bench-lib.sh" + +MODE="${1:-}" +shift || true +SUITE="swebench-verified-canary" +CACHE="$CODE_BENCH_RESULTS_ROOT/.cache" +if [[ -x "$CACHE/venv/Scripts/swebench.exe" ]]; then CLI="$CACHE/venv/Scripts/swebench.exe"; else CLI="$CACHE/venv/bin/swebench"; fi + +[[ -x "$CLI" ]] || { code_die "SWE-bench is not installed; run code-bench.sh setup --install"; exit 1; } +if command -v timeout >/dev/null 2>&1; then + timeout 10 docker info >/dev/null 2>&1 || { code_die "Docker engine is not running"; exit 1; } +else + docker info >/dev/null 2>&1 || { code_die "Docker engine is not running"; exit 1; } +fi + +case "$MODE" in + gold) + suite_json=$(code_suite_json "$SUITE") + subset=$(code_subset_path "$suite_json") + instance="${1:-$(jq -r '.instances[0].instance_id' "$subset" | tr -d '\r')}" + jq -e --arg id "$instance" '.instances[] | select(.instance_id == $id)' "$subset" >/dev/null \ + || { code_die "gold instance is outside frozen subset: $instance"; exit 1; } + run_id="gold-canary-$(date -u +%Y%m%dT%H%M%SZ)" + "$CLI" eval verified --gold -i "$instance" --run-id "$run_id" -j 1 + ;; + predictions) + predictions="${1:-}" + [[ -n "$predictions" ]] || { code_die "predictions mode needs a JSONL file"; exit 2; } + bash "$CODE_DIR/validate-predictions.sh" "$predictions" "$SUITE" + run_id="code-bench-$(date -u +%Y%m%dT%H%M%SZ)" + "$CLI" eval verified -p "$predictions" --run-id "$run_id" -j "${CODE_BENCH_EVAL_JOBS:-1}" + ;; + *) + code_die "usage: evaluate-swebench.sh gold [INSTANCE]|predictions FILE"; exit 2 + ;; +esac diff --git a/benchmarks/code/scripts/fetch-swebench-metadata.py b/benchmarks/code/scripts/fetch-swebench-metadata.py new file mode 100644 index 0000000..f23fd8e --- /dev/null +++ b/benchmarks/code/scripts/fetch-swebench-metadata.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Fetch only public SWE-bench task inputs for the frozen canary subset.""" + +from __future__ import annotations + +import argparse +import json +import urllib.parse +import urllib.request +from pathlib import Path + + +SAFE_FIELDS = ( + "instance_id", + "repo", + "base_commit", + "problem_statement", + "created_at", + "version", +) + + +def get_json(url: str) -> dict: + with urllib.request.urlopen(url, timeout=60) as response: + return json.load(response) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--subset", type=Path, required=True) + parser.add_argument("--lock", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + subset = json.loads(args.subset.read_text(encoding="utf-8")) + lock = json.loads(args.lock.read_text(encoding="utf-8")) + wanted = [item["instance_id"] for item in subset["instances"]] + wanted_set = set(wanted) + + dataset_id = lock["dataset"]["id"] + dataset_info = get_json(f"https://huggingface.co/api/datasets/{dataset_id}") + actual_revision = dataset_info.get("sha") + expected_revision = lock["dataset"]["revision"] + if actual_revision != expected_revision: + raise SystemExit( + f"dataset revision drift: expected {expected_revision}, got {actual_revision}" + ) + + api_id = urllib.parse.quote(dataset_id, safe="") + found: dict[str, dict] = {} + for offset in range(0, 500, 100): + url = ( + "https://datasets-server.huggingface.co/rows" + f"?dataset={api_id}&config=default&split=test&offset={offset}&length=100" + ) + for wrapped in get_json(url).get("rows", []): + row = wrapped["row"] + if row.get("instance_id") in wanted_set: + found[row["instance_id"]] = {key: row.get(key) for key in SAFE_FIELDS} + + missing = [instance for instance in wanted if instance not in found] + if missing: + raise SystemExit(f"missing frozen instances: {', '.join(missing)}") + + payload = { + "schema": "code-bench-public-inputs/1.0", + "dataset": lock["dataset"], + "instances": [found[instance] for instance in wanted], + } + forbidden = {"patch", "test_patch", "FAIL_TO_PASS", "PASS_TO_PASS"} + if any(forbidden.intersection(instance) for instance in payload["instances"]): + raise SystemExit("refusing to write gold or hidden-test fields") + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + print(f"WROTE: {len(wanted)} public task inputs -> {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/code/scripts/prepare-swebench-instance.sh b/benchmarks/code/scripts/prepare-swebench-instance.sh new file mode 100644 index 0000000..c5e15ab --- /dev/null +++ b/benchmarks/code/scripts/prepare-swebench-instance.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../lib/code-bench-lib.sh +source "$CODE_DIR/lib/code-bench-lib.sh" + +INSTANCE="${1:-}" +RUN_ID="${2:-}" +CONDITION="${3:-}" +[[ -n "$INSTANCE" && -n "$RUN_ID" && -n "$CONDITION" ]] || { + code_die "usage: prepare-swebench-instance.sh INSTANCE RUN_ID CONDITION"; exit 2; +} +[[ "$RUN_ID" =~ ^[A-Za-z0-9._-]+$ ]] || { code_die "unsafe run id: $RUN_ID"; exit 2; } +[[ "$CONDITION" =~ ^[A-D]$ ]] || { code_die "condition must be A, B, C, or D"; exit 2; } + +METADATA="$CODE_BENCH_RESULTS_ROOT/metadata/swebench-verified-canary.json" +[[ -f "$METADATA" ]] || { code_die "public metadata is absent; run code-bench.sh fetch-metadata"; exit 1; } +row=$(jq -ce --arg id "$INSTANCE" '.instances[] | select(.instance_id == $id)' "$METADATA") \ + || { code_die "instance is outside the frozen canary: $INSTANCE"; exit 1; } +repo=$(printf '%s' "$row" | jq -r '.repo' | tr -d '\r') +base_commit=$(printf '%s' "$row" | jq -r '.base_commit' | tr -d '\r') +[[ "$repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || { code_die "unsafe repository id: $repo"; exit 1; } +[[ "$base_commit" =~ ^[0-9a-f]{40}$ ]] || { code_die "unsafe base commit for $INSTANCE"; exit 1; } + +CELL="$CODE_BENCH_RESULTS_ROOT/runs/$RUN_ID/$INSTANCE/$CONDITION" +WORKSPACE="$CELL/workspace" +TASK_FILE="$CELL/task.md" +if [[ -e "$CELL" ]]; then + code_die "cell already exists; choose a new run id: $CELL"; exit 1 +fi +mkdir -p "$CELL" + +git clone --filter=blob:none --no-checkout "https://github.com/$repo.git" "$WORKSPACE" +git -C "$WORKSPACE" checkout --detach "$base_commit" +git -C "$WORKSPACE" status --porcelain | grep -q . && { + code_die "prepared workspace is unexpectedly dirty: $WORKSPACE"; exit 1; +} +printf '%s\n' "$row" | jq -r '.problem_statement' > "$TASK_FILE" +jq -n \ + --arg instance_id "$INSTANCE" --arg condition "$CONDITION" \ + --arg repo "$repo" --arg base_commit "$base_commit" \ + --arg workspace "$WORKSPACE" --arg task_file "$TASK_FILE" \ + '{schema:"code-bench-cell-input/1.0", instance_id:$instance_id, + condition:$condition, repo:$repo, base_commit:$base_commit, + workspace:$workspace, task_file:$task_file}' > "$CELL/input.json" +printf '%s\n' "$CELL/input.json" diff --git a/benchmarks/code/scripts/run-canary.sh b/benchmarks/code/scripts/run-canary.sh new file mode 100644 index 0000000..6efba12 --- /dev/null +++ b/benchmarks/code/scripts/run-canary.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../lib/code-bench-lib.sh +source "$CODE_DIR/lib/code-bench-lib.sh" + +RUN_ID="" +CONDITIONS="A,B,C" +TASK_LIMIT=1 +MAX_CLAUDE="" +DRY_RUN=false + +while (( $# > 0 )); do + case "$1" in + --run-id) RUN_ID="${2:?--run-id needs a value}"; shift 2 ;; + --conditions) CONDITIONS="${2:?--conditions needs a value}"; shift 2 ;; + --task-limit) TASK_LIMIT="${2:?--task-limit needs a value}"; shift 2 ;; + --max-claude-dispatches) MAX_CLAUDE="${2:?--max-claude-dispatches needs a value}"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + *) code_die "unknown canary option: $1"; exit 2 ;; + esac +done + +[[ "$RUN_ID" =~ ^[A-Za-z0-9._-]+$ ]] || { code_die "--run-id is required and must be filesystem-safe"; exit 2; } +[[ "$TASK_LIMIT" =~ ^[1-9][0-9]*$ ]] || { code_die "--task-limit must be positive"; exit 2; } +[[ "$MAX_CLAUDE" =~ ^[0-9]+$ ]] || { code_die "--max-claude-dispatches is required"; exit 2; } + +bash "$CODE_DIR/estimate-compute.sh" --suite swebench-verified-canary \ + --conditions "$CONDITIONS" --task-limit "$TASK_LIMIT" \ + --max-claude-dispatches "$MAX_CLAUDE" + +if [[ "$DRY_RUN" == true ]]; then + printf 'DRY RUN: no repositories cloned and no providers invoked.\n' + exit 0 +fi + +suite_json=$(code_suite_json "swebench-verified-canary") +subset=$(code_subset_path "$suite_json") +pred_dir="$CODE_BENCH_RESULTS_ROOT/predictions/$RUN_ID" +mkdir -p "$pred_dir" + +task_index=0 +while IFS= read -r instance; do + task_index=$((task_index + 1)) + (( task_index <= TASK_LIMIT )) || break + old_ifs=$IFS + IFS=',' + for condition in $CONDITIONS; do + input=$(bash "$CODE_DIR/scripts/prepare-swebench-instance.sh" "$instance" "$RUN_ID" "$condition") + per_condition=$(jq -r --arg id "$condition" '.conditions[] | select(.id == $id) | .dispatches.claude' \ + "$CODE_DIR/conditions.json" | tr -d '\r') + bash "$CODE_DIR/drivers/run-workflow.sh" --input "$input" \ + --predictions "$pred_dir/$condition.jsonl" \ + --max-claude-dispatches "$per_condition" + done + IFS=$old_ifs +done < <(jq -r '.instances[].instance_id' "$subset" | tr -d '\r') + +for predictions in "$pred_dir"/*.jsonl; do + bash "$CODE_DIR/validate-predictions.sh" "$predictions" swebench-verified-canary +done +printf 'COMPLETE: canary predictions -> %s\n' "$pred_dir" diff --git a/benchmarks/code/scripts/setup-swebench.sh b/benchmarks/code/scripts/setup-swebench.sh new file mode 100644 index 0000000..dbdc78f --- /dev/null +++ b/benchmarks/code/scripts/setup-swebench.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../lib/code-bench-lib.sh +source "$CODE_DIR/lib/code-bench-lib.sh" + +MODE="${1:---check}" +CACHE="$CODE_BENCH_RESULTS_ROOT/.cache" +REPO="$CACHE/SWE-bench" +VENV="$CACHE/venv" +LOCK="$CODE_DIR/external-sources.lock.json" +SHA=$(jq -r '.swebench.commit' "$LOCK" | tr -d '\r') +URL=$(jq -r '.swebench.repository' "$LOCK" | tr -d '\r') + +python_path() { + if [[ -x "$VENV/Scripts/python.exe" ]]; then printf '%s' "$VENV/Scripts/python.exe"; else printf '%s' "$VENV/bin/python"; fi +} + +cli_path() { + if [[ -x "$VENV/Scripts/swebench.exe" ]]; then printf '%s' "$VENV/Scripts/swebench.exe"; else printf '%s' "$VENV/bin/swebench"; fi +} + +check_state() { + local failed=0 + for tool in git uv docker jq; do + if command -v "$tool" >/dev/null 2>&1; then printf '%s=present\n' "$tool"; else printf '%s=missing\n' "$tool"; failed=1; fi + done + if command -v timeout >/dev/null 2>&1; then + timeout 10 docker info >/dev/null 2>&1 && engine_ready=true || engine_ready=false + else + docker info >/dev/null 2>&1 && engine_ready=true || engine_ready=false + fi + if [[ "$engine_ready" == true ]]; then printf 'docker_engine=ready\n'; else printf 'docker_engine=unavailable\n'; failed=1; fi + if [[ -d "$REPO/.git" && "$(git -C "$REPO" rev-parse HEAD 2>/dev/null)" == "$SHA" ]]; then + printf 'swebench_source=pinned\n' + else + printf 'swebench_source=not-installed\n'; failed=1 + fi + if [[ -x "$(cli_path)" ]]; then printf 'swebench_cli=ready\n'; else printf 'swebench_cli=not-installed\n'; failed=1; fi + return "$failed" +} + +case "$MODE" in + --check) + check_state + ;; + --install) + code_require git; code_require uv; code_require jq + mkdir -p "$CACHE" + if [[ ! -d "$REPO/.git" ]]; then + git clone --filter=blob:none --no-checkout "$URL" "$REPO" + fi + git -C "$REPO" fetch --depth 1 origin "$SHA" + git -C "$REPO" checkout --detach "$SHA" + if [[ ! -x "$(python_path)" ]]; then + uv venv --python 3.11 "$VENV" + fi + uv pip install --python "$(python_path)" -e "$REPO" + printf 'INSTALLED: SWE-bench %s\n' "$SHA" + check_state || true + ;; + *) + code_die "usage: setup-swebench.sh --check|--install"; exit 2 + ;; +esac diff --git a/benchmarks/code/subsets/swebench-verified-canary.json b/benchmarks/code/subsets/swebench-verified-canary.json new file mode 100644 index 0000000..2ba0bdc --- /dev/null +++ b/benchmarks/code/subsets/swebench-verified-canary.json @@ -0,0 +1,12 @@ +{ + "schema": "code-bench-subset/1.0", + "id": "swebench-verified-canary", + "selection": "Choose the shortest public problem statement within each repository, then take the five shortest across repositories in the pinned 500-task dataset; frozen before any live run.", + "instances": [ + {"instance_id": "sympy__sympy-20916", "repo": "sympy/sympy"}, + {"instance_id": "django__django-16819", "repo": "django/django"}, + {"instance_id": "scikit-learn__scikit-learn-14141", "repo": "scikit-learn/scikit-learn"}, + {"instance_id": "astropy__astropy-7166", "repo": "astropy/astropy"}, + {"instance_id": "pallets__flask-5014", "repo": "pallets/flask"} + ] +} diff --git a/benchmarks/code/suites.json b/benchmarks/code/suites.json new file mode 100644 index 0000000..b72412b --- /dev/null +++ b/benchmarks/code/suites.json @@ -0,0 +1,13 @@ +{ + "schema": "code-bench-suites/1.0", + "suites": [ + { + "id": "swebench-verified-canary", + "dataset": "princeton-nlp/SWE-bench_Verified", + "split": "test", + "subset_file": "subsets/swebench-verified-canary.json", + "task_count": 5, + "default_conditions": ["A", "B", "C", "D"] + } + ] +} diff --git a/benchmarks/code/tests/test-code-bench.sh b/benchmarks/code/tests/test-code-bench.sh new file mode 100644 index 0000000..5ae1f2d --- /dev/null +++ b/benchmarks/code/tests/test-code-bench.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +RUNNER="$CODE_DIR/code-bench.sh" +TMP=$(mktemp -d -t code-bench-test-XXXXXX) +trap 'rm -rf "$TMP"' EXIT + +TOTAL=0 +FAILED=0 +pass() { TOTAL=$((TOTAL + 1)); printf 'PASS: %s\n' "$1"; } +fail() { TOTAL=$((TOTAL + 1)); FAILED=$((FAILED + 1)); printf 'FAIL: %s\n' "$1"; } +expect_ok() { if "$@" >/dev/null 2>&1; then pass "$1"; else fail "$1"; fi; } + +if bash "$RUNNER" check >/dev/null 2>&1; then pass "manifests validate"; else fail "manifests validate"; fi + +estimate=$(bash "$RUNNER" estimate --suite swebench-verified-canary --conditions A,B,C --json 2>/dev/null) +if [[ "$(printf '%s' "$estimate" | jq -r '.cells')" == 15 \ + && "$(printf '%s' "$estimate" | jq -r '.declared_dispatches.claude')" == 20 \ + && "$(printf '%s' "$estimate" | jq -r '.declared_dispatches.codex')" == 10 \ + && "$(printf '%s' "$estimate" | jq -r '.declared_dispatches.glm')" == 5 \ + && "$(printf '%s' "$estimate" | jq -r '.declared_dispatches.kimi')" == 5 ]]; then + pass "A/B/C estimate is exact" +else + fail "A/B/C estimate is exact" +fi + +rc=0 +bash "$RUNNER" estimate --conditions A,B,C --max-claude-dispatches 4 >/dev/null 2>&1 || rc=$? +if [[ "$rc" == 75 ]]; then pass "Claude cap fails closed"; else fail "Claude cap fails closed (rc=$rc)"; fi + +cat > "$TMP/good.jsonl" <<'JSON' +{"instance_id":"sympy__sympy-20916","model_name_or_path":"condition-A","model_patch":"diff --git a/a.py b/a.py\n"} +JSON +if bash "$RUNNER" validate-predictions "$TMP/good.jsonl" >/dev/null 2>&1; then pass "valid prediction accepted"; else fail "valid prediction accepted"; fi + +cat > "$TMP/unknown.jsonl" <<'JSON' +{"instance_id":"unknown__repo-1","model_name_or_path":"condition-A","model_patch":"diff --git a/a b/a\n"} +JSON +if bash "$RUNNER" validate-predictions "$TMP/unknown.jsonl" >/dev/null 2>&1; then fail "unknown instance rejected"; else pass "unknown instance rejected"; fi + +cat > "$TMP/duplicate.jsonl" <<'JSON' +{"instance_id":"sympy__sympy-20916","model_name_or_path":"condition-A","model_patch":"diff one"} +{"instance_id":"sympy__sympy-20916","model_name_or_path":"condition-A","model_patch":"diff two"} +JSON +if bash "$RUNNER" validate-predictions "$TMP/duplicate.jsonl" >/dev/null 2>&1; then fail "duplicate prediction rejected"; else pass "duplicate prediction rejected"; fi + +cat > "$TMP/empty.jsonl" <<'JSON' +{"instance_id":"sympy__sympy-20916","model_name_or_path":"condition-A","model_patch":""} +JSON +if bash "$RUNNER" validate-predictions "$TMP/empty.jsonl" >/dev/null 2>&1; then fail "empty patch rejected"; else pass "empty patch rejected"; fi + +if grep -R -nE '"(patch|test_patch|FAIL_TO_PASS|PASS_TO_PASS)"[[:space:]]*:' \ + "$CODE_DIR/subsets" "$CODE_DIR/conditions.json" "$CODE_DIR/suites.json" >/dev/null 2>&1; then + fail "checked-in manifests contain no gold fields" +else + pass "checked-in manifests contain no gold fields" +fi + +if find "$CODE_DIR" -type f -name '*.sh' -exec grep -nE \ + '^[[:space:]]*(mapfile|readarray)([[:space:]]|$)' {} + >/dev/null 2>&1; then + fail "shell scripts are Bash 3 portable" +else + pass "shell scripts are Bash 3 portable" +fi + +TEST_RESULTS="$TMP/results" +for condition in A B C D; do + cell="$TEST_RESULTS/runs/test/sympy__sympy-20916/$condition" + mkdir -p "$cell/workspace/.git" "$TEST_RESULTS/predictions/test" + printf 'task\n' > "$cell/task.md" + jq -n --arg c "$condition" --arg w "$cell/workspace" \ + --arg t "$cell/task.md" \ + '{instance_id:"sympy__sympy-20916",condition:$c,workspace:$w,task_file:$t}' \ + > "$cell/input.json" +done + +dry_c=$(CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-workflow \ + --input "$TEST_RESULTS/runs/test/sympy__sympy-20916/C/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/C.jsonl" \ + --max-claude-dispatches 2 --dry-run 2>/dev/null) +if [[ "$(printf '%s' "$dry_c" | jq -r '.executed')" == false \ + && "$(printf '%s' "$dry_c" | jq -r '.phases | length')" == 5 \ + && "$(printf '%s' "$dry_c" | jq -r '.declared_claude_dispatches')" == 2 ]]; then + pass "condition C dry-run exposes five phases and executes nothing" +else + fail "condition C dry-run exposes five phases and executes nothing" +fi + +rc=0 +CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-workflow \ + --input "$TEST_RESULTS/runs/test/sympy__sympy-20916/D/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/D.jsonl" \ + --max-claude-dispatches 1 --dry-run >/dev/null 2>&1 || rc=$? +if [[ "$rc" == 75 ]]; then pass "workflow cap refuses condition D"; else fail "workflow cap refuses condition D (rc=$rc)"; fi + +mkdir -p "$TMP/outside/workspace/.git" +printf 'task\n' > "$TMP/outside/task.md" +jq -n --arg w "$TMP/outside/workspace" --arg t "$TMP/outside/task.md" \ + '{instance_id:"sympy__sympy-20916",condition:"A",workspace:$w,task_file:$t}' > "$TMP/outside/input.json" +if CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-workflow --input "$TMP/outside/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/A.jsonl" --max-claude-dispatches 1 --dry-run >/dev/null 2>&1; then + fail "driver rejects input outside sandbox" +else + pass "driver rejects input outside sandbox" +fi + +if bash "$RUNNER" run-canary --run-id dry-one --conditions A,B,C --task-limit 1 \ + --max-claude-dispatches 4 --dry-run >/dev/null 2>&1; then + pass "one-task A/B/C canary fits aggregate cap four" +else + fail "one-task A/B/C canary fits aggregate cap four" +fi + +rc=0 +bash "$RUNNER" run-canary --run-id dry-two --conditions A,B,C --task-limit 2 \ + --max-claude-dispatches 4 --dry-run >/dev/null 2>&1 || rc=$? +if [[ "$rc" == 75 ]]; then pass "two-task A/B/C canary exceeds aggregate cap four"; else fail "two-task A/B/C canary exceeds aggregate cap four (rc=$rc)"; fi + +printf '%d/%d assertions passed' "$((TOTAL - FAILED))" "$TOTAL" +if (( FAILED > 0 )); then printf ' (%d failed)\n' "$FAILED"; exit 1; fi +printf '\n' diff --git a/benchmarks/code/validate-predictions.sh b/benchmarks/code/validate-predictions.sh new file mode 100644 index 0000000..20379b9 --- /dev/null +++ b/benchmarks/code/validate-predictions.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/code-bench-lib.sh +source "$SCRIPT_DIR/lib/code-bench-lib.sh" + +PREDICTIONS="${1:-}" +SUITE="${2:-swebench-verified-canary}" +[[ -n "$PREDICTIONS" && -f "$PREDICTIONS" ]] || { code_die "usage: validate-predictions.sh FILE [SUITE]"; exit 2; } + +suite_json=$(code_suite_json "$SUITE") || { code_die "unknown suite: $SUITE"; exit 1; } +subset=$(code_subset_path "$suite_json") +seen=$(mktemp -t code-bench-seen-XXXXXX) +trap 'rm -f "$seen"' EXIT +line_no=0 +records=0 + +while IFS= read -r line || [[ -n "$line" ]]; do + line_no=$((line_no + 1)) + [[ -n "${line//[[:space:]]/}" ]] || continue + if ! printf '%s' "$line" | jq -e ' + type == "object" and + (.instance_id | type == "string" and length > 0) and + (.model_name_or_path | type == "string" and length > 0) and + (.model_patch | type == "string" and startswith("diff --git ")) + ' >/dev/null 2>&1; then + code_die "invalid prediction record at line $line_no"; exit 1 + fi + instance=$(printf '%s' "$line" | jq -r '.instance_id' | tr -d '\r') + jq -e --arg id "$instance" '.instances[] | select(.instance_id == $id)' "$subset" >/dev/null \ + || { code_die "prediction line $line_no names an instance outside suite $SUITE: $instance"; exit 1; } + if grep -qxF "$instance" "$seen"; then + code_die "duplicate prediction for $instance"; exit 1 + fi + printf '%s\n' "$instance" >> "$seen" + records=$((records + 1)) +done < "$PREDICTIONS" + +(( records > 0 )) || { code_die "prediction file contains no records"; exit 1; } +printf 'VALID: %s prediction record(s) for %s\n' "$records" "$SUITE" diff --git a/tests/run-all.sh b/tests/run-all.sh index 7324f79..44c94de 100755 --- a/tests/run-all.sh +++ b/tests/run-all.sh @@ -120,6 +120,7 @@ if [[ "$QUICK" == false ]]; then SUITES+=("$REPO_ROOT/benchmarks/tests/test-panel.sh") SUITES+=("$REPO_ROOT/benchmarks/tests/test-judging.sh") SUITES+=("$REPO_ROOT/benchmarks/tests/test-report.sh") + SUITES+=("$REPO_ROOT/benchmarks/code/tests/test-code-bench.sh") fi # --- run ----------------------------------------------------------------------- From 3828ce63d732950e63d971ab5fd5d62be7542366 Mon Sep 17 00:00:00 2001 From: Alan Shurafa Date: Mon, 31 Aug 2026 09:01:45 -0400 Subject: [PATCH 02/10] Fix SWE-bench evaluation on Windows Docker Co-Authored-By: Codex --- .../2026-08-31-code-benchmark-battery-plan.md | 10 ++++++ benchmarks/code/README.md | 2 ++ benchmarks/code/external-sources.lock.json | 3 +- benchmarks/code/lib/code-bench-lib.sh | 10 +++++- .../code/patches/swebench-windows-lf.patch | 14 ++++++++ benchmarks/code/scripts/evaluate-swebench.sh | 13 +++++-- benchmarks/code/scripts/setup-swebench.sh | 36 +++++++++++++++---- 7 files changed, 78 insertions(+), 10 deletions(-) create mode 100644 benchmarks/code/patches/swebench-windows-lf.patch diff --git a/.planning/notes/2026-08-31-code-benchmark-battery-plan.md b/.planning/notes/2026-08-31-code-benchmark-battery-plan.md index 4150187..1e1ddb9 100644 --- a/.planning/notes/2026-08-31-code-benchmark-battery-plan.md +++ b/.planning/notes/2026-08-31-code-benchmark-battery-plan.md @@ -55,3 +55,13 @@ tokens because Anthropic does not publish a fixed weekly token denominator. - The pinned metadata fetch contains only public task inputs, never gold data. - The official SWE-bench gold canary passes once Docker is running. - The repository aggregate test gate includes the new hermetic suite. + +## Execution result + +- Pinned SWE-bench source and CLI installed under the ignored results cache. +- Windows compatibility patch forces LF for the Linux `eval.sh` file only. +- Docker Desktop 29.6.2 official gold canary `sympy__sympy-20916` completed + and resolved 1/1 with zero infrastructure, ambiguous, or evaluator errors. +- Hermetic code-benchmark suite passed 14/14; repository aggregate passed + 42/42 after isolating the intentionally installed private Kimi key. +- No live Fable, Codex, GLM, or Kimi benchmark calls were made during setup. diff --git a/benchmarks/code/README.md b/benchmarks/code/README.md index 9395a94..d647f11 100644 --- a/benchmarks/code/README.md +++ b/benchmarks/code/README.md @@ -32,6 +32,8 @@ bash benchmarks/code/code-bench.sh setup --install These commands make no model calls. The installer pins both the official SWE-bench repository and dataset revisions from `external-sources.lock.json`. External files live under the ignored `benchmarks/results/code/` tree. +On Windows, setup also applies the tracked LF-only compatibility patch under +`patches/`; it changes only how the harness writes the Linux `eval.sh` file. ## Capped patch generation diff --git a/benchmarks/code/external-sources.lock.json b/benchmarks/code/external-sources.lock.json index 2a437d9..aaa0a5d 100644 --- a/benchmarks/code/external-sources.lock.json +++ b/benchmarks/code/external-sources.lock.json @@ -8,5 +8,6 @@ "id": "princeton-nlp/SWE-bench_Verified", "revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a", "split": "test" - } + }, + "compatibility_patches": ["patches/swebench-windows-lf.patch"] } diff --git a/benchmarks/code/lib/code-bench-lib.sh b/benchmarks/code/lib/code-bench-lib.sh index 406d320..930e749 100644 --- a/benchmarks/code/lib/code-bench-lib.sh +++ b/benchmarks/code/lib/code-bench-lib.sh @@ -44,10 +44,18 @@ code_check_manifests() { } jq -e '.schema == "code-bench-external-lock/1.0" and (.swebench.commit | test("^[0-9a-f]{40}$")) and - (.dataset.revision | test("^[0-9a-f]{40}$"))' \ + (.dataset.revision | test("^[0-9a-f]{40}$")) and + (.compatibility_patches | type == "array" and length > 0) and + (all(.compatibility_patches[]; test("^patches/[A-Za-z0-9._-]+[.]patch$")))' \ "$CODE_BENCH_DIR/external-sources.lock.json" >/dev/null || { printf 'CHECK FAIL: external-sources.lock.json\n' >&2; failures=$((failures + 1)); } + while IFS= read -r patch_rel; do + [[ -f "$CODE_BENCH_DIR/$patch_rel" ]] || { + printf 'CHECK FAIL: compatibility patch missing: %s\n' "$patch_rel" >&2 + failures=$((failures + 1)) + } + done < <(jq -r '.compatibility_patches[]' "$CODE_BENCH_DIR/external-sources.lock.json" | tr -d '\r') while IFS= read -r suite_json; do subset=$(code_subset_path "$suite_json") diff --git a/benchmarks/code/patches/swebench-windows-lf.patch b/benchmarks/code/patches/swebench-windows-lf.patch new file mode 100644 index 0000000..6700af5 --- /dev/null +++ b/benchmarks/code/patches/swebench-windows-lf.patch @@ -0,0 +1,14 @@ +diff --git a/swebench/harness/run_evaluation.py b/swebench/harness/run_evaluation.py +--- a/swebench/harness/run_evaluation.py ++++ b/swebench/harness/run_evaluation.py +@@ -359,5 +359,9 @@ def run_instance( + eval_file = Path(log_dir / "eval.sh") +- eval_file.write_text(_inject_asset_restore(test_spec.eval_script, restore_cmds)) ++ eval_file.write_text( ++ _inject_asset_restore(test_spec.eval_script, restore_cmds), ++ encoding="utf-8", ++ newline="\n", ++ ) + logger.info( + f"Eval script for {instance_id} written to {eval_file}; copying to container..." + ) diff --git a/benchmarks/code/scripts/evaluate-swebench.sh b/benchmarks/code/scripts/evaluate-swebench.sh index 1a4d1de..643c3f2 100644 --- a/benchmarks/code/scripts/evaluate-swebench.sh +++ b/benchmarks/code/scripts/evaluate-swebench.sh @@ -13,11 +13,18 @@ CACHE="$CODE_BENCH_RESULTS_ROOT/.cache" if [[ -x "$CACHE/venv/Scripts/swebench.exe" ]]; then CLI="$CACHE/venv/Scripts/swebench.exe"; else CLI="$CACHE/venv/bin/swebench"; fi [[ -x "$CLI" ]] || { code_die "SWE-bench is not installed; run code-bench.sh setup --install"; exit 1; } +# Python on Windows otherwise inherits CP1252 for pathlib.write_text(), which +# cannot encode some Unicode symbols present in official evaluation scripts. +export PYTHONUTF8=1 +export PYTHONIOENCODING=utf-8 +export HF_HUB_DISABLE_SYMLINKS_WARNING=1 if command -v timeout >/dev/null 2>&1; then timeout 10 docker info >/dev/null 2>&1 || { code_die "Docker engine is not running"; exit 1; } else docker info >/dev/null 2>&1 || { code_die "Docker engine is not running"; exit 1; } fi +EVAL_ROOT="$CODE_BENCH_RESULTS_ROOT/evaluation" +mkdir -p "$EVAL_ROOT" case "$MODE" in gold) @@ -27,14 +34,16 @@ case "$MODE" in jq -e --arg id "$instance" '.instances[] | select(.instance_id == $id)' "$subset" >/dev/null \ || { code_die "gold instance is outside frozen subset: $instance"; exit 1; } run_id="gold-canary-$(date -u +%Y%m%dT%H%M%SZ)" - "$CLI" eval verified --gold -i "$instance" --run-id "$run_id" -j 1 + (cd "$EVAL_ROOT" && "$CLI" eval verified --gold -i "$instance" --run-id "$run_id" -j 1) ;; predictions) predictions="${1:-}" [[ -n "$predictions" ]] || { code_die "predictions mode needs a JSONL file"; exit 2; } bash "$CODE_DIR/validate-predictions.sh" "$predictions" "$SUITE" + predictions_dir=$(cd "$(dirname "$predictions")" && pwd -P) + predictions="$predictions_dir/$(basename "$predictions")" run_id="code-bench-$(date -u +%Y%m%dT%H%M%SZ)" - "$CLI" eval verified -p "$predictions" --run-id "$run_id" -j "${CODE_BENCH_EVAL_JOBS:-1}" + (cd "$EVAL_ROOT" && "$CLI" eval verified -p "$predictions" --run-id "$run_id" -j "${CODE_BENCH_EVAL_JOBS:-1}") ;; *) code_die "usage: evaluate-swebench.sh gold [INSTANCE]|predictions FILE"; exit 2 diff --git a/benchmarks/code/scripts/setup-swebench.sh b/benchmarks/code/scripts/setup-swebench.sh index dbdc78f..ed640b2 100644 --- a/benchmarks/code/scripts/setup-swebench.sh +++ b/benchmarks/code/scripts/setup-swebench.sh @@ -38,6 +38,14 @@ check_state() { else printf 'swebench_source=not-installed\n'; failed=1 fi + patches_ready=true + while IFS= read -r patch_rel; do + patch_file="$CODE_DIR/$patch_rel" + if [[ ! -f "$patch_file" ]] || ! git -C "$REPO" apply --reverse --check "$patch_file" >/dev/null 2>&1; then + patches_ready=false + fi + done < <(jq -r '.compatibility_patches[]' "$LOCK" | tr -d '\r') + if [[ "$patches_ready" == true ]]; then printf 'swebench_patches=ready\n'; else printf 'swebench_patches=missing\n'; failed=1; fi if [[ -x "$(cli_path)" ]]; then printf 'swebench_cli=ready\n'; else printf 'swebench_cli=not-installed\n'; failed=1; fi return "$failed" } @@ -53,12 +61,28 @@ case "$MODE" in git clone --filter=blob:none --no-checkout "$URL" "$REPO" fi git -C "$REPO" fetch --depth 1 origin "$SHA" - git -C "$REPO" checkout --detach "$SHA" - if [[ ! -x "$(python_path)" ]]; then - uv venv --python 3.11 "$VENV" - fi - uv pip install --python "$(python_path)" -e "$REPO" - printf 'INSTALLED: SWE-bench %s\n' "$SHA" +git -C "$REPO" checkout --detach "$SHA" +while IFS= read -r patch_rel; do + patch_file="$CODE_DIR/$patch_rel" + [[ -f "$patch_file" ]] || { code_die "compatibility patch is missing: $patch_file"; exit 1; } + if git -C "$REPO" apply --reverse --check "$patch_file" >/dev/null 2>&1; then + printf 'PATCHED: %s (already applied)\n' "$patch_rel" + elif git -C "$REPO" apply --check "$patch_file"; then + git -C "$REPO" apply "$patch_file" + printf 'PATCHED: %s\n' "$patch_rel" + else + code_die "compatibility patch does not apply cleanly: $patch_rel"; exit 1 + fi +done < <(jq -r '.compatibility_patches[]' "$LOCK" | tr -d '\r') +if [[ ! -x "$(python_path)" ]]; then + uv venv --python 3.11 "$VENV" +fi +if [[ ! -x "$(cli_path)" ]]; then + uv pip install --python "$(python_path)" -e "$REPO" +else + printf 'REUSED: existing editable SWE-bench installation\n' +fi +printf 'INSTALLED: SWE-bench %s\n' "$SHA" check_state || true ;; *) From 88a49d93df65613876d17e9a77bdc833b10eb6a1 Mon Sep 17 00:00:00 2001 From: Alan Shurafa Date: Mon, 31 Aug 2026 16:51:27 -0400 Subject: [PATCH 03/10] Make code panel retries fail closed Co-Authored-By: Codex --- benchmarks/code/README.md | 4 +++ benchmarks/code/drivers/run-workflow.sh | 37 ++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/benchmarks/code/README.md b/benchmarks/code/README.md index d647f11..de38fff 100644 --- a/benchmarks/code/README.md +++ b/benchmarks/code/README.md @@ -61,6 +61,10 @@ Live phases default to medium reasoning and a 900-second timeout. Override with `CODE_BENCH_PHASE_TIMEOUT`; changing these values creates a different treatment and must be recorded in the run manifest. +`run-workflow --resume` reuses a successful Fable implementation and valid +critic artifacts. Provider-error text is rejected before the final Fable repair, +so a transient GLM/Kimi failure cannot silently degrade the four-model treatment. + ## Official evaluator Docker must be running. Validate the environment with a gold patch before diff --git a/benchmarks/code/drivers/run-workflow.sh b/benchmarks/code/drivers/run-workflow.sh index e697b5a..33a491d 100644 --- a/benchmarks/code/drivers/run-workflow.sh +++ b/benchmarks/code/drivers/run-workflow.sh @@ -12,6 +12,7 @@ INPUT_JSON="" PREDICTIONS="" MAX_CLAUDE="" DRY_RUN=false +RESUME=false CLAUDE_MODEL="${CODE_BENCH_CLAUDE_MODEL:-fable}" CODEX_MODEL_LOCAL="${CODE_BENCH_CODEX_MODEL:-gpt-5.6-sol}" CLAUDE_EFFORT_LOCAL="${CODE_BENCH_CLAUDE_EFFORT:-medium}" @@ -24,6 +25,7 @@ while (( $# > 0 )); do --predictions) PREDICTIONS="${2:?--predictions needs a value}"; shift 2 ;; --max-claude-dispatches) MAX_CLAUDE="${2:?--max-claude-dispatches needs a value}"; shift 2 ;; --dry-run) DRY_RUN=true; shift ;; + --resume) RESUME=true; shift ;; *) code_die "unknown workflow option: $1"; exit 2 ;; esac done @@ -129,6 +131,8 @@ run_fable() { command -v timeout >/dev/null 2>&1 && cmd=(timeout --foreground "${PHASE_TIMEOUT}s" "${cmd[@]}") (cd "$workspace" && "${cmd[@]}" < "$prompt") \ > "$logs/$phase.json" 2> "$logs/$phase.stderr.log" + jq -e '.type == "result" and .is_error == false and (.result | type == "string")' \ + "$logs/$phase.json" >/dev/null || { code_die "$phase did not produce a successful Claude result"; return 1; } } run_codex_repair() { @@ -151,7 +155,13 @@ run_codex_critique() { } write_implement_prompt "$cell/implement-prompt.md" -run_fable fable-implement "$cell/implement-prompt.md" +if [[ "$RESUME" == true ]] \ + && jq -e '.type == "result" and .is_error == false' "$logs/fable-implement.json" >/dev/null 2>&1 \ + && [[ -n "$(git -C "$workspace" diff --name-only)" ]]; then + printf 'REUSED: fable-implement\n' +else + run_fable fable-implement "$cell/implement-prompt.md" +fi case "$condition" in B) @@ -168,11 +178,30 @@ case "$condition" in printf '\n## ISSUE\n\n'; cat "$task_file" printf '\n## CANDIDATE PATCH\n\n'; head -c 120000 "$cell/candidate.patch" } > "$cell/critique-prompt.md" - run_codex_critique "$cell/critique-prompt.md" "$reviews/reviewer-1.md" + if [[ "$RESUME" == true && -s "$reviews/reviewer-1.md" ]] \ + && ! output_is_provider_failure "$reviews/reviewer-1.md"; then + printf 'REUSED: codex-critique\n' + else + run_codex_critique "$cell/critique-prompt.md" "$reviews/reviewer-1.md" + fi [[ -n "${ZAI_API_KEY:-}" ]] || { code_die "condition C requires ZAI_API_KEY"; exit 1; } [[ -n "${KIMI_API_KEY:-}" ]] || { code_die "condition C requires KIMI_API_KEY"; exit 1; } - invoke_glm "$cell/critique-prompt.md" "$reviews/reviewer-2.md" "$logs/glm-critique.stderr.log" false - invoke_kimi "$cell/critique-prompt.md" "$reviews/reviewer-3.md" "$logs/kimi-critique.stderr.log" + if [[ "$RESUME" != true ]] || ! validate_agent_artifact "$reviews/reviewer-2.md" "$logs/glm-critique.stderr.log" glm >/dev/null 2>&1; then + rm -f "$reviews/reviewer-2.md" + invoke_glm "$cell/critique-prompt.md" "$reviews/reviewer-2.md" "$logs/glm-critique.stderr.log" false + else + printf 'REUSED: glm-critique\n' + fi + validate_agent_artifact "$reviews/reviewer-2.md" "$logs/glm-critique.stderr.log" glm >/dev/null \ + || { code_die "GLM critique is not a valid artifact"; exit 1; } + if [[ "$RESUME" != true ]] || ! validate_agent_artifact "$reviews/reviewer-3.md" "$logs/kimi-critique.stderr.log" kimi >/dev/null 2>&1; then + rm -f "$reviews/reviewer-3.md" + invoke_kimi "$cell/critique-prompt.md" "$reviews/reviewer-3.md" "$logs/kimi-critique.stderr.log" + else + printf 'REUSED: kimi-critique\n' + fi + validate_agent_artifact "$reviews/reviewer-3.md" "$logs/kimi-critique.stderr.log" kimi >/dev/null \ + || { code_die "Kimi critique is not a valid artifact"; exit 1; } { printf '%s\n' "Re-open the current implementation and evaluate the three anonymous reviews below. Decide every finding on its merits, repair accepted issues, and run relevant tests. Do not commit." printf '\n## ISSUE\n\n'; cat "$task_file" From 735722f23df642488ada0c47ffcb1f431847fbba Mon Sep 17 00:00:00 2001 From: Alan Shurafa Date: Mon, 31 Aug 2026 18:57:24 -0400 Subject: [PATCH 04/10] Bound GLM and Kimi reasoning so capped critics return content Both providers bill reasoning tokens against max_tokens, so the 800-token critic cap spent the whole budget on reasoning and returned an empty content string. Every condition-C cell failed its artifact gate before the final repair. Z.AI rejects disabled thinking and takes reasoning_effort=low; Moonshot takes thinking:{type:disabled} at temperature 0.6. Both defaults stay unset so the document seats are unchanged. Critics now retry with a backoff, which Moonshot's org-concurrency-1 limiter needs. Co-Authored-By: Claude Opus 5 --- benchmarks/code/README.md | 9 +++++ benchmarks/code/drivers/run-workflow.sh | 39 +++++++++++++++++++--- lib/co-evolution.sh | 44 +++++++++++++++++++++---- 3 files changed, 81 insertions(+), 11 deletions(-) diff --git a/benchmarks/code/README.md b/benchmarks/code/README.md index de38fff..fbd7720 100644 --- a/benchmarks/code/README.md +++ b/benchmarks/code/README.md @@ -60,6 +60,15 @@ Live phases default to medium reasoning and a 900-second timeout. Override with `CODE_BENCH_CLAUDE_EFFORT`, `CODE_BENCH_CODEX_EFFORT`, and `CODE_BENCH_PHASE_TIMEOUT`; changing these values creates a different treatment and must be recorded in the run manifest. +Direct GLM and Kimi critiques run with bounded reasoning: GLM at +`reasoning_effort=low` (`CODE_BENCH_GLM_REASONING_EFFORT`) and Kimi with +thinking off (`CODE_BENCH_KIMI_THINKING`), under a 2500-token output cap +(`CODE_BENCH_CRITIC_MAX_TOKENS`). Both providers bill reasoning tokens against +`max_tokens`, so an unbounded critic can spend the whole budget before writing +any content and return an empty response. Each critic gets +`CODE_BENCH_CRITIC_ATTEMPTS` attempts (default 3) spaced by +`CODE_BENCH_CRITIC_RETRY_DELAY` seconds; an artifact still invalid after the +last attempt fails the cell instead of reaching the final repair. `run-workflow --resume` reuses a successful Fable implementation and valid critic artifacts. Provider-error text is rejected before the final Fable repair, diff --git a/benchmarks/code/drivers/run-workflow.sh b/benchmarks/code/drivers/run-workflow.sh index 33a491d..539cca6 100644 --- a/benchmarks/code/drivers/run-workflow.sh +++ b/benchmarks/code/drivers/run-workflow.sh @@ -18,6 +18,11 @@ CODEX_MODEL_LOCAL="${CODE_BENCH_CODEX_MODEL:-gpt-5.6-sol}" CLAUDE_EFFORT_LOCAL="${CODE_BENCH_CLAUDE_EFFORT:-medium}" CODEX_EFFORT_LOCAL="${CODE_BENCH_CODEX_EFFORT:-medium}" PHASE_TIMEOUT="${CODE_BENCH_PHASE_TIMEOUT:-900}" +CRITIC_MAX_TOKENS="${CODE_BENCH_CRITIC_MAX_TOKENS:-2500}" +GLM_CRITIC_REASONING="${CODE_BENCH_GLM_REASONING_EFFORT:-low}" +KIMI_CRITIC_THINKING="${CODE_BENCH_KIMI_THINKING:-disabled}" +CRITIC_ATTEMPTS="${CODE_BENCH_CRITIC_ATTEMPTS:-3}" +CRITIC_RETRY_DELAY="${CODE_BENCH_CRITIC_RETRY_DELAY:-15}" while (( $# > 0 )); do case "$1" in @@ -34,6 +39,9 @@ done [[ -n "$PREDICTIONS" ]] || { code_die "--predictions is required"; exit 2; } [[ "$MAX_CLAUDE" =~ ^[0-9]+$ ]] || { code_die "--max-claude-dispatches is required and must be an integer"; exit 2; } [[ "$PHASE_TIMEOUT" =~ ^[1-9][0-9]*$ ]] || { code_die "CODE_BENCH_PHASE_TIMEOUT must be positive"; exit 2; } +[[ "$CRITIC_MAX_TOKENS" =~ ^[1-9][0-9]*$ ]] || { code_die "CODE_BENCH_CRITIC_MAX_TOKENS must be positive"; exit 2; } +[[ "$CRITIC_ATTEMPTS" =~ ^[1-9][0-9]*$ ]] || { code_die "CODE_BENCH_CRITIC_ATTEMPTS must be positive"; exit 2; } +[[ "$CRITIC_RETRY_DELAY" =~ ^[0-9]+$ ]] || { code_die "CODE_BENCH_CRITIC_RETRY_DELAY must be a non-negative integer"; exit 2; } instance=$(jq -r '.instance_id' "$INPUT_JSON" | tr -d '\r') condition=$(jq -r '.condition' "$INPUT_JSON" | tr -d '\r') @@ -145,6 +153,31 @@ run_codex_repair() { < "$prompt" > "$logs/codex-repair.log" 2> "$logs/codex-repair.stderr.log" } +# GLM and Kimi both reason by default and bill reasoning against max_tokens, so +# the capped critic seats must bound reasoning too or they return an empty +# content string. Bounded retries absorb a transient provider hang without +# letting a structurally invalid artifact reach the final repair. +run_direct_critic() { + local agent="$1" out="$2" err="$3" attempt=1 + while (( attempt <= CRITIC_ATTEMPTS )); do + rm -f "$out" + case "$agent" in + glm) GLM_MAX_TOKENS="$CRITIC_MAX_TOKENS" GLM_REASONING_EFFORT="$GLM_CRITIC_REASONING" invoke_glm "$cell/critique-prompt.md" "$out" "$err" false ;; + kimi) KIMI_MAX_TOKENS="$CRITIC_MAX_TOKENS" KIMI_THINKING="$KIMI_CRITIC_THINKING" invoke_kimi "$cell/critique-prompt.md" "$out" "$err" ;; + *) code_die "unknown direct critic: $agent"; return 1 ;; + esac + if validate_agent_artifact "$out" "$err" "$agent" >/dev/null 2>&1; then + return 0 + fi + printf 'RETRY: %s critique attempt %s produced an invalid artifact\n' "$agent" "$attempt" >&2 + attempt=$((attempt + 1)) + # Moonshot enforces org concurrency 1 and answers an overlapping call + # instantly, so an immediate retry just collides again. Back off first. + if (( attempt <= CRITIC_ATTEMPTS )); then sleep "$CRITIC_RETRY_DELAY"; fi + done + return 1 +} + run_codex_critique() { local prompt="$1" out="$2" local -a cmd=(codex exec -C "$workspace" -m "$CODEX_MODEL_LOCAL" --sandbox read-only @@ -187,16 +220,14 @@ case "$condition" in [[ -n "${ZAI_API_KEY:-}" ]] || { code_die "condition C requires ZAI_API_KEY"; exit 1; } [[ -n "${KIMI_API_KEY:-}" ]] || { code_die "condition C requires KIMI_API_KEY"; exit 1; } if [[ "$RESUME" != true ]] || ! validate_agent_artifact "$reviews/reviewer-2.md" "$logs/glm-critique.stderr.log" glm >/dev/null 2>&1; then - rm -f "$reviews/reviewer-2.md" - invoke_glm "$cell/critique-prompt.md" "$reviews/reviewer-2.md" "$logs/glm-critique.stderr.log" false + run_direct_critic glm "$reviews/reviewer-2.md" "$logs/glm-critique.stderr.log" || true else printf 'REUSED: glm-critique\n' fi validate_agent_artifact "$reviews/reviewer-2.md" "$logs/glm-critique.stderr.log" glm >/dev/null \ || { code_die "GLM critique is not a valid artifact"; exit 1; } if [[ "$RESUME" != true ]] || ! validate_agent_artifact "$reviews/reviewer-3.md" "$logs/kimi-critique.stderr.log" kimi >/dev/null 2>&1; then - rm -f "$reviews/reviewer-3.md" - invoke_kimi "$cell/critique-prompt.md" "$reviews/reviewer-3.md" "$logs/kimi-critique.stderr.log" + run_direct_critic kimi "$reviews/reviewer-3.md" "$logs/kimi-critique.stderr.log" || true else printf 'REUSED: kimi-critique\n' fi diff --git a/lib/co-evolution.sh b/lib/co-evolution.sh index e221022..a1630d2 100644 --- a/lib/co-evolution.sh +++ b/lib/co-evolution.sh @@ -600,12 +600,22 @@ invoke_glm() ( local stderr_file="$3" local writable="${4:-false}" local request_file response_file config_file - local curl_rc=0 message + local curl_rc=0 message reasoning_effort [[ -n "${ZAI_API_KEY:-}" ]] || die "glm seat requires ZAI_API_KEY" [[ "$writable" == "false" ]] || die "glm seat does not support writable phases" command -v curl >/dev/null 2>&1 || die "glm seat requires curl" command -v jq >/dev/null 2>&1 || die "glm seat requires jq" + # glm-5.3-flash always reasons and Z.AI bills reasoning against max_tokens, so + # a capped caller gets finish_reason="length" with an empty content string + # unless the reasoning is bounded too. Z.AI rejects "disabled" and "medium"; + # low, high, and max are the only accepted levels. Unset keeps the provider + # default, which is what the document seats have always sent. + reasoning_effort="${GLM_REASONING_EFFORT:-}" + case "$reasoning_effort" in + ""|low|high|max) ;; + *) die "glm seat: GLM_REASONING_EFFORT must be empty, low, high, or max" ;; + esac request_file=$(mktemp -t glm-request-XXXXXX.json) response_file=$(mktemp -t glm-response-XXXXXX.json) @@ -616,12 +626,16 @@ invoke_glm() ( # incompatible with this document seat and were observed replacing a valid # response with / metadata. Z.AI's documented # Chat Completions endpoint avoids that extra agent layer entirely. - jq -Rs --arg model "${GLM_MODEL:-glm-5.3-flash}" '{ + jq -Rs --arg model "${GLM_MODEL:-glm-5.3-flash}" --argjson max_tokens "${GLM_MAX_TOKENS:-0}" \ + --arg reasoning_effort "$reasoning_effort" '{ model: $model, messages: [{role: "user", content: .}], stream: false, temperature: 0 - }' "$prompt_file" > "$request_file" + } + + (if $max_tokens > 0 then {max_tokens: $max_tokens} else {} end) + + (if $reasoning_effort != "" then {reasoning_effort: $reasoning_effort} else {} end)' \ + "$prompt_file" > "$request_file" # Keep the bearer token out of argv and logs. curl reads it from a mode-600 # temporary config file that this adapter removes on every exit path. @@ -665,11 +679,23 @@ invoke_kimi() ( local output_file="$2" local stderr_file="$3" local request_file response_file config_file - local curl_rc=0 message + local curl_rc=0 message thinking temperature [[ -n "${KIMI_API_KEY:-}" ]] || die "kimi seat requires KIMI_API_KEY" command -v curl >/dev/null 2>&1 || die "kimi seat requires curl" command -v jq >/dev/null 2>&1 || die "kimi seat requires jq" + # kimi-k3 reasons by default and Moonshot bills reasoning against max_tokens, + # so a capped caller gets finish_reason="length" with empty content. Setting + # KIMI_THINKING=disabled selects the non-thinking mode, which the API accepts + # only at temperature 0.6 (thinking mode only accepts 1). Unset is the + # provider default and leaves the document seats byte-identical. + thinking="${KIMI_THINKING:-}" + case "$thinking" in + "") temperature="${KIMI_TEMPERATURE:-1}" ;; + disabled) temperature="${KIMI_TEMPERATURE:-0.6}" ;; + *) die "kimi seat: KIMI_THINKING must be empty or disabled" ;; + esac + [[ "$temperature" =~ ^[0-9]+(\.[0-9]+)?$ ]] || die "kimi seat: KIMI_TEMPERATURE must be a number" request_file=$(mktemp -t kimi-request-XXXXXX.json) response_file=$(mktemp -t kimi-response-XXXXXX.json) @@ -678,12 +704,16 @@ invoke_kimi() ( # The document seat calls Kimi's model API directly. Kimi Code's -p agent # loop auto-runs Read/Write tools, which violates the no-tools seat boundary. - jq -Rs --arg model "${KIMI_MODEL:-kimi-k3}" '{ + jq -Rs --arg model "${KIMI_MODEL:-kimi-k3}" --argjson max_tokens "${KIMI_MAX_TOKENS:-0}" \ + --argjson temperature "$temperature" --arg thinking "$thinking" '{ model: $model, messages: [{role: "user", content: .}], stream: false, - temperature: 1 - }' "$prompt_file" > "$request_file" + temperature: $temperature + } + + (if $max_tokens > 0 then {max_tokens: $max_tokens} else {} end) + + (if $thinking != "" then {thinking: {type: $thinking}} else {} end)' \ + "$prompt_file" > "$request_file" { printf 'url = "https://api.moonshot.ai/v1/chat/completions"\n' From 5cf451c0e10d7afeee10f767caddf4c7cf511596 Mon Sep 17 00:00:00 2001 From: Alan Shurafa Date: Mon, 31 Aug 2026 18:57:31 -0400 Subject: [PATCH 05/10] Let callers override which env file supplies seat API keys The Kimi missing-key test unset KIMI_API_KEY, but the bouncer reloaded it from the repo's .env.local, so the scenario could never fail closed on a machine that has a real key on disk. CO_EVOLVE_ENV_FILE now selects the file, defaulting to .env.local, which lets the test point at a path that does not exist. The two near-identical loaders collapse into one, and its trailing guard no longer aborts the bouncer without a message when a key line has an empty value. Co-Authored-By: Claude Opus 5 --- co-evolve-bouncer.sh | 38 +++++++++++++---------------------- docs/agent-seats.md | 5 +++++ tests/kimi-seat-simulation.sh | 6 ++++++ 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/co-evolve-bouncer.sh b/co-evolve-bouncer.sh index 76b42d7..b1867d0 100644 --- a/co-evolve-bouncer.sh +++ b/co-evolve-bouncer.sh @@ -330,15 +330,19 @@ while [[ $# -gt 0 ]]; do done # Document seats may receive keys through this worktree's gitignored .env.local. -# Read only the named keys; sourcing the file would import unrelated settings into +# Read only the named key; sourcing the file would import unrelated settings into # the long-lived bouncer process. The value remains a shell variable and is # never exported — invoke_glm reads it into a mode-600 temporary curl config. -load_zai_api_key_from_env_local() { - local env_file="$SCRIPT_DIR/.env.local" +# CO_EVOLVE_ENV_FILE overrides which file is read, so a caller can point at a +# different env file — or at a path that does not exist, which is the only way +# to prove a seat fails without its key on a machine that has one on disk. +load_api_key_from_env_local() { + local name="$1" + local env_file="${CO_EVOLVE_ENV_FILE:-$SCRIPT_DIR/.env.local}" local line="" value="" - [[ -z "${ZAI_API_KEY:-}" && -r "$env_file" ]] || return 0 - line=$(grep -m 1 -E '^[[:space:]]*(export[[:space:]]+)?ZAI_API_KEY[[:space:]]*=' "$env_file" 2>/dev/null || true) + [[ -z "${!name:-}" && -r "$env_file" ]] || return 0 + line=$(grep -m 1 -E "^[[:space:]]*(export[[:space:]]+)?${name}[[:space:]]*=" "$env_file" 2>/dev/null || true) [[ -n "$line" ]] || return 0 value=$(printf '%s' "$line" | sed -e 's/^[^=]*=//' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') @@ -346,30 +350,16 @@ load_zai_api_key_from_env_local() { \"*\") value="${value#\"}"; value="${value%\"}" ;; \'*\') value="${value#\'}"; value="${value%\'}" ;; esac - [[ -n "$value" ]] && ZAI_API_KEY="$value" -} - -load_kimi_api_key_from_env_local() { - local env_file="$SCRIPT_DIR/.env.local" - local line="" value="" - - [[ -z "${KIMI_API_KEY:-}" && -r "$env_file" ]] || return 0 - line=$(grep -m 1 -E '^[[:space:]]*(export[[:space:]]+)?KIMI_API_KEY[[:space:]]*=' "$env_file" 2>/dev/null || true) - [[ -n "$line" ]] || return 0 - - value=$(printf '%s' "$line" | sed -e 's/^[^=]*=//' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') - case "$value" in - \"*\") value="${value#\"}"; value="${value%\"}" ;; - \'*\') value="${value#\'}"; value="${value%\'}" ;; - esac - [[ -n "$value" ]] && KIMI_API_KEY="$value" + if [[ -n "$value" ]]; then + printf -v "$name" '%s' "$value" + fi } # Order is intentional: a key present only in .env.local must be loaded before # the per-seat prerequisite checks. Unknown agents also fail here, before a run # directory or any paid/live compose call is created. -load_zai_api_key_from_env_local -load_kimi_api_key_from_env_local +load_api_key_from_env_local ZAI_API_KEY +load_api_key_from_env_local KIMI_API_KEY for _seat_agent in "$AGENT_A" "$AGENT_B"; do if ! seat_prereqs_ok "$_seat_agent"; then die "$SEAT_PREREQ_ERROR" diff --git a/docs/agent-seats.md b/docs/agent-seats.md index 3b57592..1a92dd9 100644 --- a/docs/agent-seats.md +++ b/docs/agent-seats.md @@ -45,6 +45,11 @@ Account creation, key entry, and interactive login remain manual steps. and add an `op://Development/...` reference to `.env.fill`. One key works on both machines. +`CO_EVOLVE_ENV_FILE` overrides which file the bouncer reads these keys from. +Point it at another env file to run against a different account, or at a path +that does not exist to check that a seat fails closed on a machine that already +has a key on disk. + ### Kimi Platform account + API key Create or log into [Kimi Platform](https://platform.kimi.ai/), add API balance, diff --git a/tests/kimi-seat-simulation.sh b/tests/kimi-seat-simulation.sh index 787be1d..20983af 100755 --- a/tests/kimi-seat-simulation.sh +++ b/tests/kimi-seat-simulation.sh @@ -9,6 +9,12 @@ BOUNCER="$REPO_ROOT/co-evolve-bouncer.sh" TEST_DIR="$(mktemp -d -t kimi-seat-XXXXXX)" trap 'rm -rf "$TEST_DIR"' EXIT +# The bouncer also reads seat keys from the repo's .env.local, which on a +# developer machine holds a real KIMI_API_KEY and would defeat the missing-key +# scenario below. Point every scenario at a path that does not exist so the +# fixture, not the machine, decides which keys are present. +export CO_EVOLVE_ENV_FILE="$TEST_DIR/absent.env" + TOTAL=0 FAILURES=0 pass() { printf 'PASS: %s\n' "$1"; } From 23f7086f417533402d52220dacbedcfa3cc3fb4a Mon Sep 17 00:00:00 2001 From: Alan Shurafa Date: Tue, 1 Sep 2026 10:27:18 -0400 Subject: [PATCH 06/10] Plan the full-suite fix and completion across both benchmark surfaces The code battery and the document batch each surfaced harness defects, and neither surface alone answers the pipeline question. One executable plan records every issue's disposition so the executing session starts from the ledger instead of rediscovering fixed problems. Co-Authored-By: Claude Opus 5 --- benchmarks/COMPLETE-SUITE-PLAN.md | 134 ++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 benchmarks/COMPLETE-SUITE-PLAN.md diff --git a/benchmarks/COMPLETE-SUITE-PLAN.md b/benchmarks/COMPLETE-SUITE-PLAN.md new file mode 100644 index 0000000..6f5c444 --- /dev/null +++ b/benchmarks/COMPLETE-SUITE-PLAN.md @@ -0,0 +1,134 @@ +# Complete Testing Suite — Fix & Finish Plan + +Drafted 2026-09-01 (Fable seat). Execute from this file in a fresh Opus session. +Scope: every benchmark surface in this repo, not just one pipeline. The two +surfaces measure different things and both ship on one results site: + +| Surface | What it measures | State today | +|---|---|---| +| S1 Code (SWE-bench battery, `benchmarks/code/`) | Do bounce/panel pipelines produce patches that resolve real issues? | Partial: B 5/5 (repair arm inert), C 4/5. A, D, solos unrun | +| S2 Documents (bounce protocol, `benchmarks/`) | Do they produce better plans/documents? Pre-registered, 3-judge | Batch b1 complete: 8x4 cells, 48 verdicts/judge. B-vs-A: no evidence (4/8). 3 judge-defect cells | +| S3 Regression (tests/run-all.sh) | Harness stays trustworthy | 42/42 green | + +## Issues ledger + +Every problem hit so far, with disposition. Fixed items stay listed so the +executing session does not re-litigate them. + +| # | Issue | Surface | Status | +|---|---|---|---| +| 1 | GLM/Kimi bill reasoning against `max_tokens`; capped critics returned empty content | S1 | FIXED `735722f` (bounded reasoning, retry+backoff) | +| 2 | kimi-seat test could not simulate a missing key on a machine with a real key | S3 | FIXED `5cf451c` (`CO_EVOLVE_ENV_FILE`) | +| 3 | Codex refuses all writes on Windows despite `--sandbox workspace-write`; B repair arm never engaged, Codex-solo blocked | S1 | OPEN — Phase 0.1 | +| 4 | Conditions A and D never run on code; solos never run | S1 | OPEN — Phase 1 | +| 5 | B scored with inert repair arm: 5/5 is really Fable-solo | S1 | OPEN — re-run after 0.1 | +| 6 | GLM/Kimi have no agent loop (chat seats only) — cannot run solo cells as-is | S1 | OPEN — Phase 1.5 | +| 7 | Judge `position_biased` verdicts discard t1/t7 primary-judge cells | S2 | OPEN — Phase 2.1 | +| 8 | `sanitize-leak` on t2 invalidates that task's comparisons | S2 | OPEN — Phase 2.1 | +| 9 | Codex judge favors B (codex-revised output) 7/0 while fable judge says 4/1 — self-preference confound | S2 | OPEN — report labeling, Phase 3 | +| 10 | Two orchestrators wrote one status file (b1 watchdog stamped `B_resolved=` into the SWE status) | S1/S2 | OPEN — Phase 0.2 | +| 11 | 5-task code subset: one task = 20 points; B/C gap is one task | S1 | OPEN — Phase 4 decides scale-up | +| 12 | HF Hub unauthenticated-rate-limit warnings during evaluation | S1 | OPEN — minor, Phase 0.3 | +| 13 | Evaluator leaves 5 images per run (`Unremoved images: 5`) | S1 | OPEN — hygiene, Phase 0.3; never auto-delete other projects' images | + +## Phase 0 — Unblock the harness + +**0.1 Codex writable workspace (the critical fix).** +Codex 0.144.5 on Windows degrades `workspace-write` to read-only (no +Landlock/Seatbelt on win32). Fix sequence, stop at the first that passes: +1. Probe `-s danger-full-access` with the existing 1-file throwaway-repo test + (prompt: replace file contents; assert the file changed). +2. If refused, probe `--dangerously-bypass-approvals-and-sandbox`. +3. If neither, route codex through WSL against the same workspace path. + +Guardrails: full access is acceptable ONLY because benchmark workspaces are +disposable clones under `benchmarks/results/code/runs/`; the driver already +diffs nothing outside the workspace. Gate the elevated flag behind +`CODE_BENCH_CODEX_SANDBOX` (default stays `workspace-write` so non-Windows +hosts keep real sandboxing). Record the mode in `run-manifest.json` — it is a +treatment-relevant fact. +Exit: driver-path probe edits a file; mode recorded in manifest. + +**0.2 Status-file single-writer.** +The b1 orchestrator's `waiting-for-swe` watchdog wrote into +`full-bc-status.txt`. Rule: one writer per status file; observers write their +own files. Add a `writer=` tag to every status line both suites emit. +Exit: grep shows tagged lines; watchdog writes `full-b1-status.txt` only. + +**0.3 Small hygiene.** Set `HF_TOKEN` via `.env.local` loader (never echo); +add a post-eval `docker image prune` scoped by the evaluator's image-name +prefix only, listed for approval as it deletes evidence-adjacent artifacts — +default OFF. + +## Phase 1 — Complete the code matrix (frozen 5-task subset) + +Order preserves pairing: never spend Fable dispatches on a condition whose +comparator cannot run. + +| Cell set | Dispatches | Est. cost | Precondition | +|---|---|---|---| +| 1.1 A (Fable solo), 5 cells | 5 Fable | ~$5 | none | +| 1.2 D (self-bounce), 5 cells | 10 Fable | ~$15-20 | none | +| 1.3 B re-run (real repair), 5 cells | 5 Fable + 5 Codex | ~$5 + plan compute | 0.1 | +| 1.4 Codex solo, 5 cells | 5 Codex | plan compute | 0.1 | +| 1.5 GLM solo + Kimi solo, single-shot tier | 10 API calls | cents | new harness | + +1.5 harness: issue text + `git grep`-selected file context in one prompt → +unified diff out → `git apply --check` gate → prediction. No tool loop, no +test execution. Label the tier "single-shot" on the site — it is a different +class of attempt and must not sit unlabeled beside agentic rows. +All cells scored by the official Docker evaluator; A reuses the existing +`prepare-instance` flow; keep `--max-claude-dispatches` caps (A=1, D=2). +Exit: every matrix row measured or explicitly marked blocked, zero +infrastructure failures, prediction files validate 5/5 unique frozen IDs. + +## Phase 2 — Close out the document suite + +**2.1 Repair the judge panel, not the generations.** Generations are frozen +and complete; only judging is defective. Fix sanitizer for the t2 leak; +re-judge t1/t2/t7 with position-counterbalanced double passes (both A/B +orders, verdict only when both orders agree; disagreement = non-decisive). +Re-emit `reports/b1.md`. Do NOT regenerate any cell — the pre-registration +forbids touching generation post-hoc. +**2.2 Judge self-preference (issue 9).** No re-run needed: the report already +never adjudicates across judges. Add the B-authorship note to the report and +site so codex-judge B-favoritism is read as a confound, not confirmation. +**2.3 Calibration baselines** for the blind judge (`evals/judge-bounce.sh`) +remain unrun — schedule as its own small batch; without them, judge scores +stay labeled uncalibrated. +Exit: 48/48 usable primary-judge pairs or documented non-decisives; report +regenerated with the same pre-registered decision rules. + +## Phase 3 — One results site for the whole suite + +Extend the published artifact (same URL) from code-only to three sections: +**Code** (the current leaderboard + new rows from Phase 1), **Documents** +(b1 pre-registered outcomes, per-judge tables, confound labels), **Harness** +(42-suite regression state, gold canary, adapter probes). One aggregator +script (`benchmarks/site/aggregate.sh`) emits a single JSON from: evaluator +report JSONs, `judge-matrix` outputs, `tests/.run-ledger`. The page reads +one data blob; no hand-edited numbers. Validity language: pre-registration +wording for S2; "frozen 5-task probe, not comparable to published SWE-bench +Verified" for S1. +Exit: site rebuilt from aggregator output alone; every number traceable to a +file on disk. + +## Phase 4 — Decide on scale, then gate + +With the full 7-row code matrix and repaired b1 report in hand, decide: +- Code: expand frozen subset (25-50 tasks) only if a pipeline-vs-solo gap + survives the 5-task probe in either direction worth confirming. +- Documents: the pre-registered held-out replication batch (b2) only if any + comparison leaves the "no evidence" band after Phase 2 re-judging. +Both are cost gates — present as go/no-go with dollar estimates, not run +autonomously. + +## Budget & sequencing + +Phase 0 is hours, no model spend beyond two codex probes. Phase 1 ≈ 20 Fable +dispatches (~$25-30 by observed per-phase costs), 10 codex cells inside the +daily guard cap, GLM/Kimi in cents. Phase 2 is judging-only (~$5-10 API). +Phases 1 and 2 run in parallel after Phase 0; Phase 3 after both; Phase 4 is +a decision, not a run. Throughout: `.env.local`, results, workspaces, and +trajectories stay uncommitted; no key values in logs; one writer per status +file; evidence never deleted. From 0922da9563a985dfb918c834017cc1ade2ea4486 Mon Sep 17 00:00:00 2001 From: Alan Shurafa Date: Tue, 1 Sep 2026 10:31:10 -0400 Subject: [PATCH 07/10] Restrict measurement to standardized public benchmarks Results on a corpus nobody outside this repo has seen are not comparable and not shareable. The bounce-protocol document benchmark is retired from measurement and all shared reporting (b1 stays archived, internal-only); SWE-bench Verified on the official pinned evaluator is the measurement surface, and future additions must be standardized public suites approved at the Phase 4 gate. Co-Authored-By: Claude Opus 5 --- benchmarks/COMPLETE-SUITE-PLAN.md | 198 +++++++++++++++--------------- 1 file changed, 99 insertions(+), 99 deletions(-) diff --git a/benchmarks/COMPLETE-SUITE-PLAN.md b/benchmarks/COMPLETE-SUITE-PLAN.md index 6f5c444..1f14e1d 100644 --- a/benchmarks/COMPLETE-SUITE-PLAN.md +++ b/benchmarks/COMPLETE-SUITE-PLAN.md @@ -1,66 +1,71 @@ # Complete Testing Suite — Fix & Finish Plan -Drafted 2026-09-01 (Fable seat). Execute from this file in a fresh Opus session. -Scope: every benchmark surface in this repo, not just one pipeline. The two -surfaces measure different things and both ship on one results site: - -| Surface | What it measures | State today | -|---|---|---| -| S1 Code (SWE-bench battery, `benchmarks/code/`) | Do bounce/panel pipelines produce patches that resolve real issues? | Partial: B 5/5 (repair arm inert), C 4/5. A, D, solos unrun | -| S2 Documents (bounce protocol, `benchmarks/`) | Do they produce better plans/documents? Pre-registered, 3-judge | Batch b1 complete: 8x4 cells, 48 verdicts/judge. B-vs-A: no evidence (4/8). 3 judge-defect cells | -| S3 Regression (tests/run-all.sh) | Harness stays trustworthy | 42/42 green | +Drafted 2026-09-01 (Fable seat), amended same day for the standardized-only +measurement policy. Execute from this file in a fresh Opus session. + +## Measurement policy (Alan, 2026-09-01 — supersedes prior scope) + +All pipeline comparison and ALL shared or published reporting uses +standardized, publicly recognized benchmarks scored by their official +evaluators — currently SWE-bench Verified on the pinned official harness. +Homegrown corpora and judge panels (the bounce-protocol document benchmark, +its 3-judge protocol, blind-judge calibration) are RETIRED from measurement +and from every shared surface. Existing internal results are archived in +place and never published. Rationale: results on a benchmark nobody outside +this repo has seen are not comparable and not worth sharing; common tests +with common baselines are. + +Boundary: the hermetic regression suite (`tests/run-all.sh`, 42 suites) is +engineering QA that gates harness correctness — it is not a benchmark, its +results are not comparison data, and it stays. + +| Surface | Status under policy | +|---|---| +| S1 Code — SWE-bench Verified battery (`benchmarks/code/`) | The measurement surface. Partial: B 5/5 (repair arm inert), C 4/5. A, D, solos unrun | +| S2 Documents — bounce-protocol suite (`benchmarks/`) | RETIRED. Batch b1 complete on disk; archive as internal evidence, no further spend, never on the shared site | +| S3 Regression — `tests/run-all.sh` | QA gate, 42/42 green. Not reported as benchmark data | ## Issues ledger -Every problem hit so far, with disposition. Fixed items stay listed so the -executing session does not re-litigate them. - -| # | Issue | Surface | Status | -|---|---|---|---| -| 1 | GLM/Kimi bill reasoning against `max_tokens`; capped critics returned empty content | S1 | FIXED `735722f` (bounded reasoning, retry+backoff) | -| 2 | kimi-seat test could not simulate a missing key on a machine with a real key | S3 | FIXED `5cf451c` (`CO_EVOLVE_ENV_FILE`) | -| 3 | Codex refuses all writes on Windows despite `--sandbox workspace-write`; B repair arm never engaged, Codex-solo blocked | S1 | OPEN — Phase 0.1 | -| 4 | Conditions A and D never run on code; solos never run | S1 | OPEN — Phase 1 | -| 5 | B scored with inert repair arm: 5/5 is really Fable-solo | S1 | OPEN — re-run after 0.1 | -| 6 | GLM/Kimi have no agent loop (chat seats only) — cannot run solo cells as-is | S1 | OPEN — Phase 1.5 | -| 7 | Judge `position_biased` verdicts discard t1/t7 primary-judge cells | S2 | OPEN — Phase 2.1 | -| 8 | `sanitize-leak` on t2 invalidates that task's comparisons | S2 | OPEN — Phase 2.1 | -| 9 | Codex judge favors B (codex-revised output) 7/0 while fable judge says 4/1 — self-preference confound | S2 | OPEN — report labeling, Phase 3 | -| 10 | Two orchestrators wrote one status file (b1 watchdog stamped `B_resolved=` into the SWE status) | S1/S2 | OPEN — Phase 0.2 | -| 11 | 5-task code subset: one task = 20 points; B/C gap is one task | S1 | OPEN — Phase 4 decides scale-up | -| 12 | HF Hub unauthenticated-rate-limit warnings during evaluation | S1 | OPEN — minor, Phase 0.3 | -| 13 | Evaluator leaves 5 images per run (`Unremoved images: 5`) | S1 | OPEN — hygiene, Phase 0.3; never auto-delete other projects' images | +| # | Issue | Status | +|---|---|---| +| 1 | GLM/Kimi bill reasoning against `max_tokens`; capped critics returned empty content | FIXED `735722f` | +| 2 | kimi-seat test could not simulate a missing key with a real key on disk | FIXED `5cf451c` | +| 3 | Codex refuses all writes on Windows despite `--sandbox workspace-write` | OPEN — Phase 0.1 | +| 4 | Conditions A and D never run on code; solos never run | OPEN — Phase 1 | +| 5 | B scored with inert repair arm: 5/5 is really Fable-solo | OPEN — re-run after 0.1 | +| 6 | GLM/Kimi have no agent loop — solo cells need a single-shot harness | OPEN — Phase 1.5 | +| 7 | Judge `position_biased` verdicts discard t1/t7 cells (doc suite) | CLOSED-RETIRED — surface withdrawn; no re-judging spend | +| 8 | `sanitize-leak` on t2 (doc suite) | CLOSED-RETIRED — same | +| 9 | Codex-judge self-preference confound (doc suite) | CLOSED-RETIRED — same | +| 10 | Two orchestrators wrote one status file (b1 watchdog stamped the SWE status) | OPEN — Phase 0.2 | +| 11 | 5-task subset: one task = 20 points; B/C gap is one task | OPEN — Phase 4 decides scale | +| 12 | HF Hub unauthenticated-rate-limit warnings during evaluation | OPEN — minor, Phase 0.3 | +| 13 | Evaluator leaves 5 images per run | OPEN — hygiene, Phase 0.3, default OFF; never delete other projects' images | ## Phase 0 — Unblock the harness **0.1 Codex writable workspace (the critical fix).** -Codex 0.144.5 on Windows degrades `workspace-write` to read-only (no -Landlock/Seatbelt on win32). Fix sequence, stop at the first that passes: -1. Probe `-s danger-full-access` with the existing 1-file throwaway-repo test - (prompt: replace file contents; assert the file changed). +Codex 0.144.5 on Windows degrades `workspace-write` to read-only. Fix +sequence, stop at the first that passes: +1. Probe `-s danger-full-access` with the existing 1-file throwaway-repo test. 2. If refused, probe `--dangerously-bypass-approvals-and-sandbox`. 3. If neither, route codex through WSL against the same workspace path. -Guardrails: full access is acceptable ONLY because benchmark workspaces are -disposable clones under `benchmarks/results/code/runs/`; the driver already -diffs nothing outside the workspace. Gate the elevated flag behind -`CODE_BENCH_CODEX_SANDBOX` (default stays `workspace-write` so non-Windows -hosts keep real sandboxing). Record the mode in `run-manifest.json` — it is a -treatment-relevant fact. +Guardrails: elevated access is acceptable ONLY because benchmark workspaces +are disposable clones under `benchmarks/results/code/runs/`. Gate behind +`CODE_BENCH_CODEX_SANDBOX` (default stays `workspace-write`); record the mode +in `run-manifest.json` — treatment-relevant fact. Exit: driver-path probe edits a file; mode recorded in manifest. -**0.2 Status-file single-writer.** -The b1 orchestrator's `waiting-for-swe` watchdog wrote into -`full-bc-status.txt`. Rule: one writer per status file; observers write their -own files. Add a `writer=` tag to every status line both suites emit. -Exit: grep shows tagged lines; watchdog writes `full-b1-status.txt` only. +**0.2 Status-file single-writer.** One writer per status file; observers get +their own files; every status line carries `writer=`. +Exit: tagged lines present; no cross-suite writes. -**0.3 Small hygiene.** Set `HF_TOKEN` via `.env.local` loader (never echo); -add a post-eval `docker image prune` scoped by the evaluator's image-name -prefix only, listed for approval as it deletes evidence-adjacent artifacts — -default OFF. +**0.3 Small hygiene.** `HF_TOKEN` via the `.env.local` loader (never echo). +Image-prune stays default OFF. -## Phase 1 — Complete the code matrix (frozen 5-task subset) +## Phase 1 — Complete the code matrix (SWE-bench Verified, frozen 5-task subset) Order preserves pairing: never spend Fable dispatches on a condition whose comparator cannot run. @@ -74,61 +79,56 @@ comparator cannot run. | 1.5 GLM solo + Kimi solo, single-shot tier | 10 API calls | cents | new harness | 1.5 harness: issue text + `git grep`-selected file context in one prompt → -unified diff out → `git apply --check` gate → prediction. No tool loop, no -test execution. Label the tier "single-shot" on the site — it is a different -class of attempt and must not sit unlabeled beside agentic rows. -All cells scored by the official Docker evaluator; A reuses the existing -`prepare-instance` flow; keep `--max-claude-dispatches` caps (A=1, D=2). -Exit: every matrix row measured or explicitly marked blocked, zero -infrastructure failures, prediction files validate 5/5 unique frozen IDs. - -## Phase 2 — Close out the document suite - -**2.1 Repair the judge panel, not the generations.** Generations are frozen -and complete; only judging is defective. Fix sanitizer for the t2 leak; -re-judge t1/t2/t7 with position-counterbalanced double passes (both A/B -orders, verdict only when both orders agree; disagreement = non-decisive). -Re-emit `reports/b1.md`. Do NOT regenerate any cell — the pre-registration -forbids touching generation post-hoc. -**2.2 Judge self-preference (issue 9).** No re-run needed: the report already -never adjudicates across judges. Add the B-authorship note to the report and -site so codex-judge B-favoritism is read as a confound, not confirmation. -**2.3 Calibration baselines** for the blind judge (`evals/judge-bounce.sh`) -remain unrun — schedule as its own small batch; without them, judge scores -stay labeled uncalibrated. -Exit: 48/48 usable primary-judge pairs or documented non-decisives; report -regenerated with the same pre-registered decision rules. - -## Phase 3 — One results site for the whole suite - -Extend the published artifact (same URL) from code-only to three sections: -**Code** (the current leaderboard + new rows from Phase 1), **Documents** -(b1 pre-registered outcomes, per-judge tables, confound labels), **Harness** -(42-suite regression state, gold canary, adapter probes). One aggregator -script (`benchmarks/site/aggregate.sh`) emits a single JSON from: evaluator -report JSONs, `judge-matrix` outputs, `tests/.run-ledger`. The page reads -one data blob; no hand-edited numbers. Validity language: pre-registration -wording for S2; "frozen 5-task probe, not comparable to published SWE-bench -Verified" for S1. +unified diff → `git apply --check` gate → prediction. Label the tier +"single-shot" everywhere — never unlabeled beside agentic rows. +All cells scored by the official Docker evaluator; caps A=1, D=2 on +`--max-claude-dispatches`. +Exit: every matrix row measured or explicitly blocked; zero infrastructure +failures; prediction files validate 5/5 unique frozen IDs. + +## Phase 2 — Retire the homegrown document benchmark + +No model spend. Archive-only: +1. Leave batch b1 results and `reports/b1.md` in place as internal evidence; + they are never published, linked, or summarized on any shared surface. +2. Add a retirement note to `benchmarks/README.md` (doc-suite root): retired + from measurement 2026-09-01 per standardized-only policy; direct readers + to `benchmarks/code/` for the active benchmark. +3. Cancel outstanding doc-suite work: t1/t2/t7 re-judging, sanitizer fix for + judging, blind-judge calibration baselines. Do not delete any code or + results — retire, don't destroy. +Exit: retirement note committed; no doc-suite job scheduled anywhere. + +## Phase 3 — Results site: standardized benchmarks only + +Update the existing artifact (same URL). Two sections: +1. **Leaderboard** — SWE-bench Verified frozen-subset matrix, all Phase 1 + rows, coverage labels, per-task dots. +2. **Methodology & integrity** — evaluator pin + gold canary 1/1, dispatch + counts, per-condition cost, harness commit, and the standing caveat: + frozen 5-task probe, not comparable to published full-500 scores. +Remove nothing that is already standardized; add no homegrown-benchmark +content. One aggregator script (`benchmarks/site/aggregate.sh`) builds a +single JSON from evaluator reports + run logs; the page renders only that. Exit: site rebuilt from aggregator output alone; every number traceable to a -file on disk. +file on disk; zero references to the retired suite. -## Phase 4 — Decide on scale, then gate +## Phase 4 — Scale gate (go/no-go recommendation, never autonomous) -With the full 7-row code matrix and repaired b1 report in hand, decide: -- Code: expand frozen subset (25-50 tasks) only if a pipeline-vs-solo gap - survives the 5-task probe in either direction worth confirming. -- Documents: the pre-registered held-out replication batch (b2) only if any - comparison leaves the "no evidence" band after Phase 2 re-judging. -Both are cost gates — present as go/no-go with dollar estimates, not run -autonomously. +Present with dollar estimates, run nothing: +- Expand the SWE-bench Verified subset (25-50 tasks) if any pipeline-vs-solo + gap from Phase 1 is worth confirming. +- Candidate additional suites — standardized public benchmarks only, each + with an official pinned harness (e.g. SWE-bench Lite, Terminal-Bench, + Aider Polyglot, LiveCodeBench). No internal corpus is ever proposed. +- Note for the document pipeline: it currently has NO standardized public + benchmark. Until one exists and is adopted at this gate, document-pipeline + quality claims stay unmeasured rather than internally measured. ## Budget & sequencing -Phase 0 is hours, no model spend beyond two codex probes. Phase 1 ≈ 20 Fable -dispatches (~$25-30 by observed per-phase costs), 10 codex cells inside the -daily guard cap, GLM/Kimi in cents. Phase 2 is judging-only (~$5-10 API). -Phases 1 and 2 run in parallel after Phase 0; Phase 3 after both; Phase 4 is -a decision, not a run. Throughout: `.env.local`, results, workspaces, and -trajectories stay uncommitted; no key values in logs; one writer per status -file; evidence never deleted. +Phase 0 is hours, two codex probes. Phase 1 ≈ 20 Fable dispatches (~$25-30), +10 codex cells inside the daily guard cap, GLM/Kimi in cents. Phase 2 is a +docs commit. Phase 3 after Phase 1. Phase 4 is a decision. Throughout: +`.env.local`, results, workspaces, trajectories stay uncommitted; no key +values in logs; one writer per status file; evidence never deleted. From ccfb22660cde19c566bd1accdbf7e09df4f7bc28 Mon Sep 17 00:00:00 2001 From: Alan Shurafa Date: Tue, 1 Sep 2026 12:58:27 -0400 Subject: [PATCH 08/10] Retire the homegrown document benchmark from measurement Results on a corpus and judge panel that exist only in this repository cannot be compared against anything anyone else has run, so they are not worth publishing. Measurement and every shared surface now use standardized public benchmarks scored by their official evaluators. Batch b1 stays on disk as internal evidence. Nothing is deleted; the runbook remains for reading archived batches. Outstanding doc-suite work (re-judging, sanitizer fix, judge calibration) is cancelled, not deferred. Co-Authored-By: Claude Opus 5 --- benchmarks/README.md | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 9c760ce..bd3673a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,8 +1,31 @@ # Co-Evolution Benchmark Suite -Repository-patch and coding-agent evaluation lives in -[`benchmarks/code/`](code/README.md). It uses official deterministic evaluators -and a separate compute-cap contract; this document continues to describe the +## Retired from measurement, 2026-09-01 + +The plan-composition benchmark described below no longer measures anything. +Measurement and every shared report now use standardized, publicly recognized +benchmarks scored by their official evaluators — currently SWE-bench Verified +on the pinned official harness in [`benchmarks/code/`](code/README.md). Go +there for the active benchmark. + +A result on a corpus and a judge panel that exist only in this repository +cannot be compared against anything anyone else has run, so it is not worth +publishing. The batch b1 results and `reports/b1.md` stay on disk as internal +evidence of what was built; they are never published, linked, or summarized on +a shared surface. The outstanding work on this suite — re-judging the +position-biased cells, the sanitizer fix for judging, and blind-judge +calibration baselines — is cancelled rather than deferred. + +Nothing here is deleted. The runbook below still describes what the scripts do +if you need to read or re-derive an archived batch. Do not schedule new +batches, and do not add this suite's numbers to any published page. + +A future benchmark may be added only if it is a standardized public suite with +an official pinned evaluator. + +--- + +The rest of this document is the archived runbook for the retired plan-composition benchmark. Batch runbook for comparing plan-composition conditions (solo Fable, Codex From 8339d5289a04cac2821fbb6d67a628f43dcce056 Mon Sep 17 00:00:00 2001 From: Alan Shurafa Date: Tue, 1 Sep 2026 12:58:37 -0400 Subject: [PATCH 09/10] Complete the SWE-bench matrix with solo arms and a writable repair arm Codex 0.144.5 on Windows accepts --sandbox workspace-write and then runs read-only, so condition B's repair step reviewed every patch and changed nothing: its 5/5 was Fable's first draft with a discarded review attached. Gate the mode behind CODE_BENCH_CODEX_SANDBOX and record what each cell actually used, because the mode decides what the treatment is. Add the arms that make the pipeline rows interpretable: codex-solo, and a single-shot tier for GLM and Kimi, which have no agent loop here. Tier is a declared field so a one-prompt answer is never ranked beside a coding agent unlabelled. The single-shot gate applies with --recount: chat models miscount @@ headers while proposing sound edits, and the agentic arms never write a hunk header at all. One status file, one writer, every line tagged - two orchestrators once interleaved into a timeline that described neither run. Co-Authored-By: Claude Opus 5 --- .gitignore | 4 + benchmarks/code/README.md | 82 ++- benchmarks/code/code-bench.sh | 4 + benchmarks/code/conditions.json | 25 + benchmarks/code/drivers/run-single-shot.sh | 215 ++++++ benchmarks/code/drivers/run-workflow.sh | 41 +- benchmarks/code/lib/code-bench-lib.sh | 57 ++ benchmarks/code/scripts/evaluate-swebench.sh | 4 + benchmarks/code/scripts/extract-diff.sh | 41 ++ .../code/scripts/prepare-swebench-instance.sh | 4 +- benchmarks/code/scripts/select-context.py | 137 ++++ benchmarks/code/suites.json | 2 +- benchmarks/code/tests/test-code-bench.sh | 156 +++- benchmarks/site/aggregate.sh | 33 + benchmarks/site/build-site-data.py | 398 ++++++++++ benchmarks/site/render-page.py | 678 ++++++++++++++++++ 16 files changed, 1860 insertions(+), 21 deletions(-) create mode 100644 benchmarks/code/drivers/run-single-shot.sh create mode 100644 benchmarks/code/scripts/extract-diff.sh create mode 100644 benchmarks/code/scripts/select-context.py create mode 100644 benchmarks/site/aggregate.sh create mode 100644 benchmarks/site/build-site-data.py create mode 100644 benchmarks/site/render-page.py diff --git a/.gitignore b/.gitignore index 79222ac..51da9a5 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,7 @@ tests/.run-ledger/ # Machine-local operational notes (private paths, spend, secrets workflow) .planning/local/ + +# Python bytecode caches +__pycache__/ +*.pyc diff --git a/benchmarks/code/README.md b/benchmarks/code/README.md index fbd7720..5036112 100644 --- a/benchmarks/code/README.md +++ b/benchmarks/code/README.md @@ -17,8 +17,27 @@ and [Harbor Terminal-Bench](https://www.harborframework.com/docs/tutorials/runni - **Future lanes:** Aider Polyglot, Terminal-Bench, and a private recent-issue set can plug into the same prediction and reporting contract. -The four conditions are declared in `conditions.json`. Condition D is retained -as a self-bounce control even when the product question focuses on A/B/C. +The conditions are declared in `conditions.json`, each with a `tier`: + +| Id | Label | Tier | What runs | +|---|---|---|---| +| A | fable-solo | agentic | Fable implements once | +| B | cross-vendor-bounce | agentic | Fable implements; Codex repairs | +| C | fable-led-panel | agentic | Fable implements; Codex/GLM/Kimi critique; Fable repairs | +| D | fable-self-bounce | agentic | Fable implements, then reviews and repairs its own patch | +| E | codex-solo | agentic | Codex implements once | +| F | glm-solo-single-shot | single-shot | GLM sees the issue plus retrieved context, returns one diff | +| G | kimi-solo-single-shot | single-shot | Kimi sees the issue plus retrieved context, returns one diff | + +D is retained as a self-bounce control even when the product question focuses +on A/B/C. E is the comparator that makes B's repair arm interpretable. + +**The tier is not cosmetic.** Agentic conditions run a coding agent with file +tools and test execution. GLM and Kimi are reachable here only as chat +completions, so F and G get one prompt and one answer: no file reads, no test +runs, no second look. A single-shot number is not a like-for-like result +against an agentic one and must never be reported beside one without the +label. ## Zero-compute setup @@ -56,6 +75,53 @@ the final repair decision. If `ANTHROPIC_API_KEY` is present, live generation fails closed so Claude Console credits cannot be charged accidentally instead of the Max subscription. +### Codex sandbox mode + +Codex 0.144.5 on Windows accepts `--sandbox workspace-write` and then reports +`sandbox: read-only`, so every write is refused and a repair arm silently goes +inert while still producing a plausible review. `CODE_BENCH_CODEX_SANDBOX` +selects the mode; it defaults to `workspace-write` and the driver records the +value it used in each cell's `run-manifest.json`, because the mode changes what +the treatment actually is. + +```bash +CODE_BENCH_CODEX_SANDBOX=danger-full-access bash benchmarks/code/code-bench.sh run-workflow ... +``` + +Elevated access is defensible only because a benchmark workspace is a +throwaway clone under the ignored `benchmarks/results/code/runs/` tree. Do not +set it for anything else. + +### Single-shot tier + +```bash +bash benchmarks/code/code-bench.sh run-single-shot \ + --input "$input" \ + --predictions benchmarks/results/code/predictions/matrix/F.jsonl \ + --agent glm +``` + +`scripts/select-context.py` picks the files the prompt shows, deterministically +and from public inputs only: paths named in the issue rank first, then files +matched by the issue's rarest identifiers, with tests and examples at half +weight. The model returns a unified diff, `scripts/extract-diff.sh` pulls it +out of the prose, and `git apply --check --recount` gates it; a rejected diff is +fed back with the apply error for up to `CODE_BENCH_SINGLE_SHOT_ATTEMPTS` tries +(default 3). `--recount` recomputes the `@@` line counts from the hunk body and +changes no line of the proposed edit. Without it the gate scores the model's +line arithmetic rather than its patch, which is an artefact of asking for a diff +at all: the agentic conditions edit files directly and never write a hunk +header. Measured on the first two GLM cells, the strict gate rejected every +attempt while `--recount` accepted the first. + +A cell that never produces an applicable patch writes `outcome.json` and +contributes no prediction rather than a broken one. + +`CODE_BENCH_SINGLE_SHOT_MAX_TOKENS` defaults to 32000. Both providers bill +reasoning against `max_tokens`, and on this prompt shape GLM spends roughly +19k reasoning tokens before it writes anything, so a smaller budget returns +`finish_reason=length` with empty content every single time. + Live phases default to medium reasoning and a 900-second timeout. Override with `CODE_BENCH_CLAUDE_EFFORT`, `CODE_BENCH_CODEX_EFFORT`, and `CODE_BENCH_PHASE_TIMEOUT`; changing these values creates a different treatment @@ -94,6 +160,18 @@ Prediction JSONL uses the official SWE-bench fields: Gold patches and hidden tests are never stored in this repository or supplied to a generation workflow. +## Results site + +```bash +bash benchmarks/site/aggregate.sh +``` + +Builds `benchmarks/results/code/site/leaderboard.json` from the evaluator +reports, the evaluator's own per-instance verdicts, and the run manifests. Each +row carries the report file it came from and each task carries the +`report.json` that decided it, so every number on the published page can be +checked against a file on disk. The page renders that JSON and nothing else. + ## Compute contract `estimate` reports declared provider dispatches. For the five-task canary: diff --git a/benchmarks/code/code-bench.sh b/benchmarks/code/code-bench.sh index 931402b..4f608db 100644 --- a/benchmarks/code/code-bench.sh +++ b/benchmarks/code/code-bench.sh @@ -31,6 +31,9 @@ case "$COMMAND" in run-workflow) exec bash "$SCRIPT_DIR/drivers/run-workflow.sh" "$@" ;; + run-single-shot) + exec bash "$SCRIPT_DIR/drivers/run-single-shot.sh" "$@" + ;; run-canary) exec bash "$SCRIPT_DIR/scripts/run-canary.sh" "$@" ;; @@ -53,6 +56,7 @@ usage: code-bench.sh COMMAND [options] setup --check|--install inspect or install pinned SWE-bench tooling prepare-instance ID RUN COND clone a clean public-input workspace run-workflow [options] generate one capped condition prediction + run-single-shot [options] generate one single-shot tier prediction run-canary [options] run a batch with one aggregate Claude cap validate-predictions FILE validate JSONL before official scoring gold-canary [INSTANCE] verify the official evaluator with a gold patch diff --git a/benchmarks/code/conditions.json b/benchmarks/code/conditions.json index d1ce7be..bb48f73 100644 --- a/benchmarks/code/conditions.json +++ b/benchmarks/code/conditions.json @@ -4,26 +4,51 @@ { "id": "A", "label": "fable-solo", + "tier": "agentic", "description": "Fable investigates, edits, and tests once.", "dispatches": {"claude": 1, "codex": 0, "glm": 0, "kimi": 0} }, { "id": "B", "label": "cross-vendor-bounce", + "tier": "agentic", "description": "Fable implements; Codex reviews and repairs the patch.", "dispatches": {"claude": 1, "codex": 1, "glm": 0, "kimi": 0} }, { "id": "C", "label": "fable-led-panel", + "tier": "agentic", "description": "Fable implements; Codex, GLM, and Kimi critique; Fable repairs.", "dispatches": {"claude": 2, "codex": 1, "glm": 1, "kimi": 1} }, { "id": "D", "label": "fable-self-bounce", + "tier": "agentic", "description": "Fable implements and performs a second review/repair pass.", "dispatches": {"claude": 2, "codex": 0, "glm": 0, "kimi": 0} + }, + { + "id": "E", + "label": "codex-solo", + "tier": "agentic", + "description": "Codex investigates, edits, and tests once.", + "dispatches": {"claude": 0, "codex": 1, "glm": 0, "kimi": 0} + }, + { + "id": "F", + "label": "glm-solo-single-shot", + "tier": "single-shot", + "description": "GLM receives the issue plus retrieved file context and returns one unified diff. No agent loop, no test execution.", + "dispatches": {"claude": 0, "codex": 0, "glm": 1, "kimi": 0} + }, + { + "id": "G", + "label": "kimi-solo-single-shot", + "tier": "single-shot", + "description": "Kimi receives the issue plus retrieved file context and returns one unified diff. No agent loop, no test execution.", + "dispatches": {"claude": 0, "codex": 0, "glm": 0, "kimi": 1} } ] } diff --git a/benchmarks/code/drivers/run-single-shot.sh b/benchmarks/code/drivers/run-single-shot.sh new file mode 100644 index 0000000..e34b825 --- /dev/null +++ b/benchmarks/code/drivers/run-single-shot.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +# Single-shot patch generation for models that have no coding-agent loop. +# +# GLM and Kimi are reachable here only as chat completions: no tools, no file +# reads, no test execution. Their cells therefore cannot be a like-for-like +# comparison against the agentic conditions, and the tier is labelled +# "single-shot" in the manifest, the prediction record, and every report so a +# reader never sees these rows unqualified beside an agentic row. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CODE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=../lib/code-bench-lib.sh +source "$CODE_DIR/lib/code-bench-lib.sh" +# shellcheck source=../../../lib/co-evolution.sh +source "$CODE_BENCH_REPO_ROOT/lib/co-evolution.sh" + +INPUT_JSON="" +PREDICTIONS="" +AGENT="" +DRY_RUN=false +# Both providers bill reasoning against max_tokens. Measured on this prompt +# shape, GLM spends ~19k reasoning tokens before writing a patch, so an 8k +# budget returns finish_reason=length with empty content every time. +MAX_TOKENS="${CODE_BENCH_SINGLE_SHOT_MAX_TOKENS:-32000}" +ATTEMPTS="${CODE_BENCH_SINGLE_SHOT_ATTEMPTS:-3}" +RETRY_DELAY="${CODE_BENCH_SINGLE_SHOT_RETRY_DELAY:-15}" +CONTEXT_FILES="${CODE_BENCH_SINGLE_SHOT_CONTEXT_FILES:-6}" +CONTEXT_BYTES="${CODE_BENCH_SINGLE_SHOT_CONTEXT_BYTES:-24000}" +GLM_CRITIC_REASONING="${CODE_BENCH_GLM_REASONING_EFFORT:-low}" +KIMI_CRITIC_THINKING="${CODE_BENCH_KIMI_THINKING:-disabled}" + +while (( $# > 0 )); do + case "$1" in + --input) INPUT_JSON="${2:?--input needs a value}"; shift 2 ;; + --predictions) PREDICTIONS="${2:?--predictions needs a value}"; shift 2 ;; + --agent) AGENT="${2:?--agent needs a value}"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + *) code_die "unknown single-shot option: $1"; exit 2 ;; + esac +done + +[[ -f "$INPUT_JSON" ]] || { code_die "--input must name a prepared input.json"; exit 2; } +[[ -n "$PREDICTIONS" ]] || { code_die "--predictions is required"; exit 2; } +case "$AGENT" in glm|kimi) ;; *) code_die "--agent must be glm or kimi"; exit 2 ;; esac +for numeric in MAX_TOKENS ATTEMPTS CONTEXT_FILES CONTEXT_BYTES; do + [[ "${!numeric}" =~ ^[1-9][0-9]*$ ]] || { code_die "$numeric must be a positive integer"; exit 2; } +done +[[ "$RETRY_DELAY" =~ ^[0-9]+$ ]] || { code_die "RETRY_DELAY must be a non-negative integer"; exit 2; } + +instance=$(jq -r '.instance_id' "$INPUT_JSON" | tr -d '\r') +condition=$(jq -r '.condition' "$INPUT_JSON" | tr -d '\r') +workspace=$(jq -r '.workspace' "$INPUT_JSON" | tr -d '\r') +task_file=$(jq -r '.task_file' "$INPUT_JSON" | tr -d '\r') +[[ -d "$workspace/.git" && -f "$task_file" ]] || { code_die "prepared workspace or task file is missing"; exit 1; } + +condition_json=$(jq -ce --arg id "$condition" '.conditions | map(select(.id == $id)) | if length == 1 then .[0] else empty end' "$CODE_DIR/conditions.json") \ + || { code_die "unknown condition: $condition"; exit 1; } +tier=$(printf '%s' "$condition_json" | jq -r '.tier' | tr -d '\r') +label=$(printf '%s' "$condition_json" | jq -r '.label' | tr -d '\r') +[[ "$tier" == "single-shot" ]] || { code_die "condition $condition is tier $tier; this driver only runs single-shot cells"; exit 1; } +declared=$(printf '%s' "$condition_json" | jq -r --arg a "$AGENT" '.dispatches[$a]' | tr -d '\r') +(( declared == 1 )) || { code_die "condition $condition does not declare a $AGENT dispatch"; exit 1; } + +results_root=$(cd "$CODE_BENCH_RESULTS_ROOT" && pwd -P) +input_dir=$(cd "$(dirname "$INPUT_JSON")" && pwd -P) +input_abs="$input_dir/$(basename "$INPUT_JSON")" +workspace_abs=$(cd "$workspace" && pwd -P) +task_dir=$(cd "$(dirname "$task_file")" && pwd -P) +task_abs="$task_dir/$(basename "$task_file")" +case "$input_abs" in "$results_root"/runs/*/*/*/input.json) ;; *) code_die "input.json is outside the benchmark run sandbox"; exit 1 ;; esac +[[ "$workspace_abs" == "$input_dir/workspace" ]] || { code_die "workspace does not belong to the prepared cell"; exit 1; } +[[ "$task_abs" == "$input_dir/task.md" ]] || { code_die "task file does not belong to the prepared cell"; exit 1; } +mkdir -p "$(dirname "$PREDICTIONS")" +pred_dir=$(cd "$(dirname "$PREDICTIONS")" && pwd -P) +pred_abs="$pred_dir/$(basename "$PREDICTIONS")" +case "$pred_abs" in "$results_root"/predictions/*) ;; *) code_die "predictions path is outside the benchmark prediction sandbox"; exit 1 ;; esac + +if [[ "$DRY_RUN" == true ]]; then + jq -n --arg instance "$instance" --arg condition "$condition" --arg agent "$AGENT" \ + --arg tier "$tier" --arg label "$label" \ + '{instance:$instance,condition:$condition,agent:$agent,tier:$tier,label:$label, + phases:["select-context","single-shot-diff","git-apply-gate"],executed:false}' + exit 0 +fi + +code_load_env_key ZAI_API_KEY +code_load_env_key KIMI_API_KEY +case "$AGENT" in + glm) [[ -n "${ZAI_API_KEY:-}" ]] || { code_die "condition $condition requires ZAI_API_KEY"; exit 1; } ;; + kimi) [[ -n "${KIMI_API_KEY:-}" ]] || { code_die "condition $condition requires KIMI_API_KEY"; exit 1; } ;; +esac + +cell="$input_dir" +logs="$cell/logs" +mkdir -p "$logs" + +git -C "$workspace" diff --quiet || { code_die "workspace is already dirty; prepare a fresh cell"; exit 1; } + +context_list="$cell/context-files.txt" +python "$CODE_DIR/scripts/select-context.py" \ + --workspace "$workspace" --task "$task_file" --max-files "$CONTEXT_FILES" \ + 2> "$logs/select-context.stderr.log" | tr -d '\r' > "$context_list" \ + || { code_die "context selection failed; see $logs/select-context.stderr.log"; exit 1; } +[[ -s "$context_list" ]] || { code_die "context selection returned no files"; exit 1; } + +model_name="" +case "$AGENT" in + glm) model_name="${GLM_MODEL:-glm-5.3-flash}" ;; + kimi) model_name="${KIMI_MODEL:-kimi-k3}" ;; +esac +jq -n --arg instance "$instance" --arg condition "$condition" --arg agent "$AGENT" \ + --arg tier "$tier" --arg label "$label" --arg model "$model_name" \ + --argjson max_tokens "$MAX_TOKENS" --argjson attempts "$ATTEMPTS" \ + --argjson context_files "$CONTEXT_FILES" --argjson context_bytes "$CONTEXT_BYTES" \ + --rawfile context "$context_list" \ + '{schema:"code-bench-single-shot/1.0",instance:$instance,condition:$condition, + tier:$tier,label:$label,agent:$agent,model:$model, + output_max_tokens:$max_tokens,apply_attempts:$attempts, + retrieval:{max_files:$context_files,max_bytes_per_file:$context_bytes, + selected:($context|split("\n")|map(select(length>0)))}}' \ + > "$cell/run-manifest.json" + +write_prompt() { + local out="$1" feedback="$2" + { + printf '%s\n' "You are fixing a bug in a Python repository. You cannot run commands, open files, or execute tests: everything you may use is below." + printf '%s\n' "Return exactly one unified diff and nothing else, inside a single fenced block that opens with three backticks followed by diff." + printf '%s\n' "Rules for the diff: use git-style headers (diff --git a/PATH b/PATH), keep paths relative to the repository root, include @@ hunk headers with at least three lines of unchanged context, and change only what the issue requires." + printf '%s\n' "Do not add or modify tests. Do not reformat unrelated code. Do not invent files that are not shown." + printf '\n## ISSUE\n\n' + cat "$task_file" + printf '\n## REPOSITORY FILES\n\n' + while IFS= read -r rel; do + [[ -n "$rel" ]] || continue + printf '### %s\n\n' "$rel" + printf '%s\n' '```python' + head -c "$CONTEXT_BYTES" "$workspace/$rel" + printf '\n%s\n\n' '```' + done < <(tr -d '\r' < "$context_list") + if [[ -n "$feedback" && -s "$feedback" ]]; then + printf '\n## YOUR PREVIOUS ATTEMPT DID NOT APPLY\n\n' + printf '%s\n' "git apply rejected the diff below. Produce a corrected diff whose context lines match the files above exactly." + printf '\n### git apply error\n\n' + head -c 4000 "$feedback" + printf '\n' + fi + } > "$out" +} + +invoke_single_shot() { + local prompt="$1" out="$2" err="$3" + case "$AGENT" in + glm) GLM_MAX_TOKENS="$MAX_TOKENS" GLM_REASONING_EFFORT="$GLM_CRITIC_REASONING" \ + invoke_glm "$prompt" "$out" "$err" false ;; + kimi) KIMI_MAX_TOKENS="$MAX_TOKENS" KIMI_THINKING="$KIMI_CRITIC_THINKING" \ + invoke_kimi "$prompt" "$out" "$err" ;; + esac +} + +feedback="" +attempt=1 +applied=false +while (( attempt <= ATTEMPTS )); do + response="$logs/$AGENT-response-$attempt.md" + stderr_log="$logs/$AGENT-response-$attempt.stderr.log" + candidate="$cell/candidate-$attempt.patch" + apply_log="$logs/git-apply-$attempt.log" + write_prompt "$cell/single-shot-prompt-$attempt.md" "$feedback" + invoke_single_shot "$cell/single-shot-prompt-$attempt.md" "$response" "$stderr_log" + + if ! validate_agent_artifact "$response" "$stderr_log" "$AGENT" >/dev/null 2>&1; then + printf 'RETRY: %s attempt %s returned no usable response\n' "$AGENT" "$attempt" >&2 + printf 'provider returned no usable response\n' > "$apply_log" + elif ! bash "$CODE_DIR/scripts/extract-diff.sh" "$response" "$candidate" 2>/dev/null; then + printf 'RETRY: %s attempt %s contained no unified diff\n' "$AGENT" "$attempt" >&2 + printf 'no unified diff found in the response\n' > "$apply_log" + elif git -C "$workspace" apply --check --recount --whitespace=nowarn "$candidate" > "$apply_log" 2>&1; then + git -C "$workspace" apply --recount --whitespace=nowarn "$candidate" >> "$apply_log" 2>&1 \ + || { code_die "git apply --check passed but apply failed; see $apply_log"; exit 1; } + applied=true + break + else + printf 'RETRY: %s attempt %s produced a diff git apply rejected\n' "$AGENT" "$attempt" >&2 + fi + + feedback="$apply_log" + attempt=$((attempt + 1)) + if (( attempt <= ATTEMPTS )); then sleep "$RETRY_DELAY"; fi +done + +if [[ "$applied" != true ]]; then + jq -n --arg instance "$instance" --arg condition "$condition" --arg agent "$AGENT" \ + --arg tier "$tier" --argjson attempts "$ATTEMPTS" \ + '{schema:"code-bench-single-shot-outcome/1.0",instance:$instance,condition:$condition, + agent:$agent,tier:$tier,outcome:"no-applicable-patch",attempts:$attempts}' \ + > "$cell/outcome.json" + code_die "$AGENT produced no applicable patch for $instance after $ATTEMPTS attempts" + exit 1 +fi + +patch="$cell/final.patch" +git -C "$workspace" diff --binary > "$patch" +[[ -s "$patch" ]] || { code_die "single-shot run produced an empty patch"; exit 1; } +record="$cell/prediction.json" +jq -n --arg instance_id "$instance" --arg model "co-evolution-condition-$condition" \ + --rawfile model_patch "$patch" \ + '{instance_id:$instance_id,model_name_or_path:$model,model_patch:$model_patch}' > "$record" +jq -c . "$record" >> "$PREDICTIONS" +jq -n --arg instance "$instance" --arg condition "$condition" --arg agent "$AGENT" \ + --arg tier "$tier" --argjson attempts "$attempt" \ + '{schema:"code-bench-single-shot-outcome/1.0",instance:$instance,condition:$condition, + agent:$agent,tier:$tier,outcome:"patch-applied",attempts:$attempts}' \ + > "$cell/outcome.json" +printf 'WROTE: %s\n' "$record" diff --git a/benchmarks/code/drivers/run-workflow.sh b/benchmarks/code/drivers/run-workflow.sh index 539cca6..91b2bb2 100644 --- a/benchmarks/code/drivers/run-workflow.sh +++ b/benchmarks/code/drivers/run-workflow.sh @@ -23,6 +23,7 @@ GLM_CRITIC_REASONING="${CODE_BENCH_GLM_REASONING_EFFORT:-low}" KIMI_CRITIC_THINKING="${CODE_BENCH_KIMI_THINKING:-disabled}" CRITIC_ATTEMPTS="${CODE_BENCH_CRITIC_ATTEMPTS:-3}" CRITIC_RETRY_DELAY="${CODE_BENCH_CRITIC_RETRY_DELAY:-15}" +CODEX_SANDBOX=$(code_codex_sandbox) || exit 2 while (( $# > 0 )); do case "$1" in @@ -76,6 +77,8 @@ case "$condition" in B) phases="fable-implement,codex-repair" ;; C) phases="fable-implement,codex-critique,glm-critique,kimi-critique,fable-repair" ;; D) phases="fable-implement,fable-self-repair" ;; + E) phases="codex-implement" ;; + F|G) code_die "single-shot conditions run through run-single-shot.sh"; exit 2 ;; esac if [[ "$DRY_RUN" == true ]]; then jq -n --arg instance "$instance" --arg condition "$condition" --arg phases "$phases" \ @@ -91,20 +94,8 @@ fi command -v claude >/dev/null 2>&1 || { code_die "claude CLI is required"; exit 1; } command -v codex >/dev/null 2>&1 || { code_die "codex CLI is required"; exit 1; } -load_named_key() { - local name="$1" env_file="$CODE_BENCH_REPO_ROOT/.env.local" line="" value="" - [[ -z "${!name:-}" && -r "$env_file" ]] || return 0 - line=$(grep -m 1 -E "^[[:space:]]*(export[[:space:]]+)?${name}[[:space:]]*=" "$env_file" 2>/dev/null || true) - [[ -n "$line" ]] || return 0 - value=$(printf '%s' "$line" | sed -e 's/^[^=]*=//' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') - case "$value" in - \"*\") value="${value#\"}"; value="${value%\"}" ;; - \'*\') value="${value#\'}"; value="${value%\'}" ;; - esac - [[ -n "$value" ]] && printf -v "$name" '%s' "$value" -} -load_named_key ZAI_API_KEY -load_named_key KIMI_API_KEY +code_load_env_key ZAI_API_KEY +code_load_env_key KIMI_API_KEY cell="$input_dir" logs="$cell/logs" @@ -113,10 +104,12 @@ mkdir -p "$logs" "$reviews" jq -n --arg instance "$instance" --arg condition "$condition" \ --arg claude_model "$CLAUDE_MODEL" --arg claude_effort "$CLAUDE_EFFORT_LOCAL" \ --arg codex_model "$CODEX_MODEL_LOCAL" --arg codex_effort "$CODEX_EFFORT_LOCAL" \ + --arg codex_sandbox "$CODEX_SANDBOX" \ --argjson phase_timeout "$PHASE_TIMEOUT" --argjson declared_claude "$claude_needed" \ '{schema:"code-bench-run/1.0",instance:$instance,condition:$condition, models:{claude:$claude_model,codex:$codex_model}, effort:{claude:$claude_effort,codex:$codex_effort}, + sandbox:{codex:$codex_sandbox}, phase_timeout_seconds:$phase_timeout,declared_claude_dispatches:$declared_claude}' \ > "$cell/run-manifest.json" @@ -145,7 +138,7 @@ run_fable() { run_codex_repair() { local prompt="$1" - local -a cmd=(codex exec -C "$workspace" -m "$CODEX_MODEL_LOCAL" --sandbox workspace-write + local -a cmd=(codex exec -C "$workspace" -m "$CODEX_MODEL_LOCAL" --sandbox "$CODEX_SANDBOX" --ephemeral --ignore-user-config -c approval_policy="never" -c model_reasoning_effort="$CODEX_EFFORT_LOCAL" -) command -v timeout >/dev/null 2>&1 && cmd=(timeout --foreground "${PHASE_TIMEOUT}s" "${cmd[@]}") @@ -153,6 +146,16 @@ run_codex_repair() { < "$prompt" > "$logs/codex-repair.log" 2> "$logs/codex-repair.stderr.log" } +run_codex_implement() { + local prompt="$1" + local -a cmd=(codex exec -C "$workspace" -m "$CODEX_MODEL_LOCAL" --sandbox "$CODEX_SANDBOX" + --ephemeral --ignore-user-config -c approval_policy="never" + -c model_reasoning_effort="$CODEX_EFFORT_LOCAL" -) + command -v timeout >/dev/null 2>&1 && cmd=(timeout --foreground "${PHASE_TIMEOUT}s" "${cmd[@]}") + "${cmd[@]}" \ + < "$prompt" > "$logs/codex-implement.log" 2> "$logs/codex-implement.stderr.log" +} + # GLM and Kimi both reason by default and bill reasoning against max_tokens, so # the capped critic seats must bound reasoning too or they return an empty # content string. Bounded retries absorb a transient provider hang without @@ -188,7 +191,13 @@ run_codex_critique() { } write_implement_prompt "$cell/implement-prompt.md" -if [[ "$RESUME" == true ]] \ +if [[ "$condition" == E ]]; then + if [[ "$RESUME" == true && -n "$(git -C "$workspace" diff --name-only)" ]]; then + printf 'REUSED: codex-implement\n' + else + run_codex_implement "$cell/implement-prompt.md" + fi +elif [[ "$RESUME" == true ]] \ && jq -e '.type == "result" and .is_error == false' "$logs/fable-implement.json" >/dev/null 2>&1 \ && [[ -n "$(git -C "$workspace" diff --name-only)" ]]; then printf 'REUSED: fable-implement\n' diff --git a/benchmarks/code/lib/code-bench-lib.sh b/benchmarks/code/lib/code-bench-lib.sh index 930e749..cc2520c 100644 --- a/benchmarks/code/lib/code-bench-lib.sh +++ b/benchmarks/code/lib/code-bench-lib.sh @@ -32,6 +32,8 @@ code_check_manifests() { (.conditions | length > 0) and ([.conditions[].id] | length == (unique | length)) and (all(.conditions[]; (.dispatches | keys) == ["claude","codex","glm","kimi"])) and + (all(.conditions[]; (.label | type == "string" and length > 0))) and + (all(.conditions[]; .tier == "agentic" or .tier == "single-shot")) and ([.conditions[].dispatches[] | type == "number" and . >= 0 and floor == .] | all)' \ "$CODE_BENCH_DIR/conditions.json" >/dev/null || { printf 'CHECK FAIL: conditions.json\n' >&2; failures=$((failures + 1)); @@ -88,3 +90,58 @@ code_check_manifests() { (( failures == 0 )) || return 1 printf 'CHECK: code benchmark manifests PASS\n' } + +# Codex 0.144.5 on Windows degrades `--sandbox workspace-write` to read-only, +# so the write phases refuse every edit and a repair arm silently goes inert. +# The mode is a treatment-relevant fact, so it is a gated variable rather than +# a constant: elevated access is acceptable only because benchmark workspaces +# are disposable clones under the ignored results tree. +code_codex_sandbox() { + local mode="${CODE_BENCH_CODEX_SANDBOX:-workspace-write}" + case "$mode" in + read-only|workspace-write|danger-full-access) ;; + *) code_die "CODE_BENCH_CODEX_SANDBOX must be read-only, workspace-write, or danger-full-access"; return 1 ;; + esac + printf '%s' "$mode" +} + +# Reads one key from the seat env file without ever echoing its value. Honours +# CO_EVOLVE_ENV_FILE so a test can point at a fixture instead of the real file. +code_load_env_key() { + local name="$1" env_file line value + env_file="${CO_EVOLVE_ENV_FILE:-$CODE_BENCH_REPO_ROOT/.env.local}" + [[ -z "${!name:-}" && -r "$env_file" ]] || return 0 + line=$(grep -m 1 -E "^[[:space:]]*(export[[:space:]]+)?${name}[[:space:]]*=" "$env_file" 2>/dev/null || true) + [[ -n "$line" ]] || return 0 + value=$(printf '%s' "$line" | sed -e 's/^[^=]*=//' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + case "$value" in + \"*\") value="${value#\"}"; value="${value%\"}" ;; + \'*\') value="${value#\'}"; value="${value%\'}" ;; + esac + [[ -n "$value" ]] && printf -v "$name" '%s' "$value" +} + +# Two orchestrators once appended to the same status file and the resulting +# timeline described neither run. A status file now belongs to exactly one +# writer: the owner is recorded on creation, every line carries writer=, and a +# second writer is refused instead of interleaving. +code_status_init() { + local file="$1" writer="$2" + [[ -n "$file" && -n "$writer" ]] || { code_die "code_status_init needs FILE WRITER"; return 1; } + [[ "$writer" =~ ^[A-Za-z0-9._-]+$ ]] || { code_die "writer id must be filesystem-safe: $writer"; return 1; } + mkdir -p "$(dirname "$file")" + if [[ -f "$file.writer" ]] && [[ "$(cat "$file.writer")" != "$writer" ]]; then + code_die "status file $file already belongs to writer $(cat "$file.writer"); use your own file" + return 1 + fi + printf '%s' "$writer" > "$file.writer" + printf 'writer=%s state=running started=%s\n' "$writer" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$file" +} + +code_status_append() { + local file="$1" writer="$2"; shift 2 + [[ -f "$file.writer" ]] || { code_die "status file $file has no owner; call code_status_init first"; return 1; } + [[ "$(cat "$file.writer")" == "$writer" ]] \ + || { code_die "writer $writer may not append to $file (owner $(cat "$file.writer"))"; return 1; } + printf 'writer=%s at=%s %s\n' "$writer" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >> "$file" +} diff --git a/benchmarks/code/scripts/evaluate-swebench.sh b/benchmarks/code/scripts/evaluate-swebench.sh index 643c3f2..f55c036 100644 --- a/benchmarks/code/scripts/evaluate-swebench.sh +++ b/benchmarks/code/scripts/evaluate-swebench.sh @@ -18,6 +18,10 @@ if [[ -x "$CACHE/venv/Scripts/swebench.exe" ]]; then CLI="$CACHE/venv/Scripts/sw export PYTHONUTF8=1 export PYTHONIOENCODING=utf-8 export HF_HUB_DISABLE_SYMLINKS_WARNING=1 +# Unauthenticated Hub reads are rate-limited and the warning fires mid-run. A +# token is optional; it is read from the seat env file and never echoed. +code_load_env_key HF_TOKEN +if [[ -n "${HF_TOKEN:-}" ]]; then export HF_TOKEN; fi if command -v timeout >/dev/null 2>&1; then timeout 10 docker info >/dev/null 2>&1 || { code_die "Docker engine is not running"; exit 1; } else diff --git a/benchmarks/code/scripts/extract-diff.sh b/benchmarks/code/scripts/extract-diff.sh new file mode 100644 index 0000000..09ca6d2 --- /dev/null +++ b/benchmarks/code/scripts/extract-diff.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Pull the unified diff out of a chat response. +# +# Single-shot seats answer in prose-plus-fence, and the fence label varies +# (```diff, ```patch, or bare ```). Some answers skip the fence and trail +# explanation after the patch. This reads the response and writes just the +# diff; it exits non-zero when the response contains no git-style diff at all, +# which is a cell failure rather than a patch. +set -euo pipefail + +RESPONSE="${1:-}" +OUT="${2:-}" +[[ -f "$RESPONSE" && -n "$OUT" ]] || { printf 'usage: extract-diff.sh RESPONSE OUT\n' >&2; exit 2; } + +raw="${OUT}.raw" +awk ' + /^[[:space:]]*```/ { + if (inblock) { exit } + if (seen == 0) { inblock = 1; seen = 1; next } + next + } + inblock { print } +' "$RESPONSE" > "$raw" +grep -q '^diff --git ' "$raw" || cp "$RESPONSE" "$raw" + +# Keep the run of diff lines and drop the prose on either side of it. Anything +# that is not recognisable diff syntax ends the patch. +awk ' + BEGIN { started = 0 } + !started && /^diff --git / { started = 1 } + started { + if ($0 ~ /^(diff --git |index |--- |\+\+\+ |@@ |[+-]|[[:space:]]|\\ No newline|new file mode |deleted file mode |old mode |new mode |similarity index |rename from |rename to |Binary files |GIT binary patch)/ || $0 == "") { + print + } else { + exit + } + } +' "$raw" > "$OUT" +rm -f "$raw" + +grep -q '^diff --git ' "$OUT" diff --git a/benchmarks/code/scripts/prepare-swebench-instance.sh b/benchmarks/code/scripts/prepare-swebench-instance.sh index c5e15ab..e4ebf7d 100644 --- a/benchmarks/code/scripts/prepare-swebench-instance.sh +++ b/benchmarks/code/scripts/prepare-swebench-instance.sh @@ -13,7 +13,9 @@ CONDITION="${3:-}" code_die "usage: prepare-swebench-instance.sh INSTANCE RUN_ID CONDITION"; exit 2; } [[ "$RUN_ID" =~ ^[A-Za-z0-9._-]+$ ]] || { code_die "unsafe run id: $RUN_ID"; exit 2; } -[[ "$CONDITION" =~ ^[A-D]$ ]] || { code_die "condition must be A, B, C, or D"; exit 2; } +[[ "$CONDITION" =~ ^[A-Za-z0-9_-]+$ ]] || { code_die "unsafe condition id: $CONDITION"; exit 2; } +jq -e --arg id "$CONDITION" 'any(.conditions[]; .id == $id)' "$CODE_DIR/conditions.json" >/dev/null \ + || { code_die "unknown condition: $CONDITION"; exit 2; } METADATA="$CODE_BENCH_RESULTS_ROOT/metadata/swebench-verified-canary.json" [[ -f "$METADATA" ]] || { code_die "public metadata is absent; run code-bench.sh fetch-metadata"; exit 1; } diff --git a/benchmarks/code/scripts/select-context.py b/benchmarks/code/scripts/select-context.py new file mode 100644 index 0000000..f80ab8e --- /dev/null +++ b/benchmarks/code/scripts/select-context.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Pick the repository files a single-shot patch attempt should see. + +The single-shot tier has no agent loop: the model cannot open files, so the +harness must choose the context. Selection is deterministic and uses only +public inputs (the issue text and the repository at its base commit) so the +same cell always builds the same prompt. +""" +import argparse +import re +import subprocess +import sys + +SOURCE_SUFFIXES = ('.py', '.pyx', '.pyi') + +# Words that appear in nearly every issue report and match nearly every file. +STOPWORDS = frozenset(""" +about above after again against all also always analysis and another any are +around because been before being below between both build call called calls +can case cases change changed changes check code coming compare could current +currently data default depending description details different does doing done +during each either else error errors even every example expect expected +failing fails false first fixed follow following from function functions +generate get gets github given gives group handle has have here how however +implementation import instead into issue itself just keep known last later +like line lines list little look make makes many maybe method methods might +more most much must name names need needs never new none not note now number +object objects only open option options order other output outside over own +part pass patch please point possible previous print problem produce python +raise raised raises rather read really reason regression release report +reported reproduce result results return returns run running same seems self +set should show shown side similar simple since small some something still +such support sure take test tests than that the their them then there these +they thing think this those three through time trying two type types under +until update use used user uses using value values version very want was way +what when where whether which while will with within without work working +would write wrong your +""".split()) + +FILE_PATH_RE = re.compile(r'\b((?:[\w.-]+/)+[\w.-]+\.(?:py|pyx|pyi))\b') +BACKTICK_RE = re.compile(r'`([^`\n]{2,120})`') +IDENT_RE = re.compile(r'\b([A-Za-z_][A-Za-z0-9_]{3,})\b') + + +def git(workspace, *args): + proc = subprocess.run(('git', '-C', workspace) + args, + capture_output=True, text=True, errors='replace') + return proc.returncode, proc.stdout + + +def tracked_source_files(workspace): + rc, out = git(workspace, 'ls-files') + if rc != 0: + sys.exit('ERROR: git ls-files failed in %s' % workspace) + return set(p for p in out.splitlines() if p.endswith(SOURCE_SUFFIXES)) + + +def candidate_tokens(issue, limit): + """Rank issue tokens: backticked and dotted names first, then identifiers.""" + weighted = {} + + def add(token, weight): + token = token.strip() + if len(token) < 4 or token.lower() in STOPWORDS: + return + weighted[token] = max(weighted.get(token, 0.0), weight) + + for span in BACKTICK_RE.findall(issue): + add(span, 3.0) + for part in re.split(r'[^A-Za-z0-9_]+', span): + add(part, 2.0) + for ident in IDENT_RE.findall(issue): + add(ident, 1.0) + + ordered = sorted(weighted.items(), key=lambda kv: (-kv[1], kv[0])) + return [token for token, _ in ordered[:limit]] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--workspace', required=True) + ap.add_argument('--task', required=True) + ap.add_argument('--max-files', type=int, default=6) + ap.add_argument('--max-tokens', type=int, default=40) + args = ap.parse_args() + + # Bash callers read this list line by line; Windows text mode would append + # a carriage return to every path and break the lookups downstream. + sys.stdout.reconfigure(newline=chr(10)) + + with open(args.task, encoding='utf-8', errors='replace') as handle: + issue = handle.read() + + tracked = tracked_source_files(args.workspace) + if not tracked: + sys.exit('ERROR: workspace has no tracked Python sources') + + scores = {} + + # A path spelled out in the issue is the strongest possible signal. + for path in FILE_PATH_RE.findall(issue): + for candidate in tracked: + if candidate == path or candidate.endswith('/' + path): + scores[candidate] = scores.get(candidate, 0.0) + 10.0 + + for token in candidate_tokens(issue, args.max_tokens): + rc, out = git(args.workspace, 'grep', '-l', '-F', '--', token) + if rc != 0: + continue + hits = [p for p in out.splitlines() if p in tracked] + # A token matching half the repository says nothing about where the bug + # is; a token matching three files says a great deal. + if not hits or len(hits) > 40: + continue + share = 1.0 / len(hits) + for path in hits: + scores[path] = scores.get(path, 0.0) + share + + if not scores: + sys.exit('ERROR: no candidate files matched the issue text') + + # Test and example modules are legitimate context but they crowd out the + # source file the patch has to touch, so they compete at half weight. + for path in list(scores): + parts = path.split('/') + is_support = ('tests' in parts or 'test' in parts + or 'examples' in parts or parts[-1].startswith('test_')) + if is_support: + scores[path] *= 0.5 + + ranked = sorted(scores.items(), key=lambda kv: (-kv[1], kv[0])) + for path, _ in ranked[:args.max_files]: + print(path) + + +if __name__ == '__main__': + main() diff --git a/benchmarks/code/suites.json b/benchmarks/code/suites.json index b72412b..23ef5db 100644 --- a/benchmarks/code/suites.json +++ b/benchmarks/code/suites.json @@ -7,7 +7,7 @@ "split": "test", "subset_file": "subsets/swebench-verified-canary.json", "task_count": 5, - "default_conditions": ["A", "B", "C", "D"] + "default_conditions": ["A", "B", "C", "D", "E", "F", "G"] } ] } diff --git a/benchmarks/code/tests/test-code-bench.sh b/benchmarks/code/tests/test-code-bench.sh index 5ae1f2d..ba00cfc 100644 --- a/benchmarks/code/tests/test-code-bench.sh +++ b/benchmarks/code/tests/test-code-bench.sh @@ -66,7 +66,7 @@ else fi TEST_RESULTS="$TMP/results" -for condition in A B C D; do +for condition in A B C D E F; do cell="$TEST_RESULTS/runs/test/sympy__sympy-20916/$condition" mkdir -p "$cell/workspace/.git" "$TEST_RESULTS/predictions/test" printf 'task\n' > "$cell/task.md" @@ -118,6 +118,160 @@ bash "$RUNNER" run-canary --run-id dry-two --conditions A,B,C --task-limit 2 \ --max-claude-dispatches 4 --dry-run >/dev/null 2>&1 || rc=$? if [[ "$rc" == 75 ]]; then pass "two-task A/B/C canary exceeds aggregate cap four"; else fail "two-task A/B/C canary exceeds aggregate cap four (rc=$rc)"; fi +if jq -e 'all(.conditions[]; (.tier == "agentic") or (.tier == "single-shot"))' \ + "$CODE_DIR/conditions.json" >/dev/null 2>&1; then + pass "every condition declares a tier" +else + fail "every condition declares a tier" +fi + +if jq -e '[.conditions[] | select(.tier == "single-shot")] + | length > 0 and all(.[]; .dispatches.claude == 0 and .dispatches.codex == 0)' \ + "$CODE_DIR/conditions.json" >/dev/null 2>&1; then + pass "single-shot conditions spend no agentic dispatch" +else + fail "single-shot conditions spend no agentic dispatch" +fi + +refusal=$(CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-workflow \ + --input "$TEST_RESULTS/runs/test/sympy__sympy-20916/F/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/F.jsonl" \ + --max-claude-dispatches 0 --dry-run 2>&1 >/dev/null) +rc=$? +if (( rc != 0 )) && printf '%s' "$refusal" | grep -q 'run-single-shot.sh'; then + pass "agentic driver refuses a single-shot condition" +else + fail "agentic driver refuses a single-shot condition (rc=$rc)" +fi + +dry_f=$(CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-single-shot \ + --input "$TEST_RESULTS/runs/test/sympy__sympy-20916/F/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/F.jsonl" \ + --agent glm --dry-run 2>/dev/null) +if [[ "$(printf '%s' "$dry_f" | jq -r '.tier')" == "single-shot" \ + && "$(printf '%s' "$dry_f" | jq -r '.executed')" == false ]]; then + pass "single-shot dry-run reports its tier and executes nothing" +else + fail "single-shot dry-run reports its tier and executes nothing" +fi + +if CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-single-shot \ + --input "$TEST_RESULTS/runs/test/sympy__sympy-20916/F/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/F.jsonl" \ + --agent kimi --dry-run >/dev/null 2>&1; then + fail "single-shot driver rejects an undeclared agent" +else + pass "single-shot driver rejects an undeclared agent" +fi + +if CODE_BENCH_RESULTS_ROOT="$TEST_RESULTS" bash "$RUNNER" run-single-shot \ + --input "$TEST_RESULTS/runs/test/sympy__sympy-20916/A/input.json" \ + --predictions "$TEST_RESULTS/predictions/test/A.jsonl" \ + --agent glm --dry-run >/dev/null 2>&1; then + fail "single-shot driver rejects an agentic condition" +else + pass "single-shot driver rejects an agentic condition" +fi + +EXTRACT="$CODE_DIR/scripts/extract-diff.sh" +FIX="$TMP/fixture" +mkdir -p "$FIX" +git -C "$FIX" init -q +printf 'alpha\nbeta\ngamma\n' > "$FIX/sample.txt" +git -C "$FIX" add sample.txt >/dev/null 2>&1 +git -C "$FIX" -c user.email=t@e -c user.name=t commit -qm seed >/dev/null 2>&1 +printf 'alpha\nBETA\ngamma\n' > "$FIX/sample.txt" +git -C "$FIX" diff > "$TMP/real.patch" +git -C "$FIX" checkout -- sample.txt + +{ + printf 'Here is the fix you asked for.\n\n' + printf '%s\n' '```diff' + cat "$TMP/real.patch" + printf '%s\n' '```' + printf '\nLet me know if you want tests as well.\n' +} > "$TMP/fenced-response.md" +if bash "$EXTRACT" "$TMP/fenced-response.md" "$TMP/fenced.patch" >/dev/null 2>&1 \ + && git -C "$FIX" apply --check "$TMP/fenced.patch" >/dev/null 2>&1; then + pass "fenced diff extracts and applies" +else + fail "fenced diff extracts and applies" +fi + +{ + cat "$TMP/real.patch" + printf 'That should resolve the reported behaviour.\n' +} > "$TMP/bare-response.md" +if bash "$EXTRACT" "$TMP/bare-response.md" "$TMP/bare.patch" >/dev/null 2>&1 \ + && git -C "$FIX" apply --check "$TMP/bare.patch" >/dev/null 2>&1 \ + && ! grep -q 'reported behaviour' "$TMP/bare.patch"; then + pass "unfenced diff extracts without trailing prose" +else + fail "unfenced diff extracts without trailing prose" +fi + +printf 'I could not reproduce the issue.\n' > "$TMP/no-diff.md" +if bash "$EXTRACT" "$TMP/no-diff.md" "$TMP/none.patch" >/dev/null 2>&1; then + fail "response without a diff is rejected" +else + pass "response without a diff is rejected" +fi + +# The single-shot gate applies with --recount because chat models routinely get +# the @@ line counts wrong while proposing a correct edit. Strict apply rejects +# such a patch; --recount accepts it without altering a single edited line. +miscounted="$TMP/miscounted.patch" +sed 's/^@@ -1,3 +1,3 @@/@@ -1,9 +1,9 @@/' "$TMP/real.patch" > "$miscounted" +strict_rc=0 +git -C "$FIX" apply --check "$miscounted" >/dev/null 2>&1 || strict_rc=$? +recount_rc=0 +git -C "$FIX" apply --check --recount "$miscounted" >/dev/null 2>&1 || recount_rc=$? +if (( strict_rc != 0 )) && (( recount_rc == 0 )); then + pass "--recount rescues a miscounted hunk header" +else + fail "--recount rescues a miscounted hunk header (strict=$strict_rc recount=$recount_rc)" +fi + +if grep -q 'apply --check --recount' "$CODE_DIR/drivers/run-single-shot.sh"; then + pass "single-shot driver gates with --recount" +else + fail "single-shot driver gates with --recount" +fi + +STATUS_TEST="$TMP/status/battery.txt" +if ( set -e + source "$CODE_DIR/lib/code-bench-lib.sh" + code_status_init "$STATUS_TEST" alpha + code_status_append "$STATUS_TEST" alpha "cells=1/5" + ) >/dev/null 2>&1 && grep -q '^writer=alpha .*cells=1/5' "$STATUS_TEST"; then + pass "status lines carry their writer" +else + fail "status lines carry their writer" +fi + +if ( source "$CODE_DIR/lib/code-bench-lib.sh" + code_status_append "$STATUS_TEST" beta "cells=2/5" + ) >/dev/null 2>&1; then + fail "a second writer cannot append to another writer's status file" +else + pass "a second writer cannot append to another writer's status file" +fi + +sandbox_default=$( source "$CODE_DIR/lib/code-bench-lib.sh"; code_codex_sandbox ) +if [[ "$sandbox_default" == "workspace-write" ]]; then + pass "codex sandbox defaults to workspace-write" +else + fail "codex sandbox defaults to workspace-write" +fi + +if ( source "$CODE_DIR/lib/code-bench-lib.sh" + CODE_BENCH_CODEX_SANDBOX=wide-open code_codex_sandbox + ) >/dev/null 2>&1; then + fail "codex sandbox rejects an unknown mode" +else + pass "codex sandbox rejects an unknown mode" +fi + printf '%d/%d assertions passed' "$((TOTAL - FAILED))" "$TOTAL" if (( FAILED > 0 )); then printf ' (%d failed)\n' "$FAILED"; exit 1; fi printf '\n' diff --git a/benchmarks/site/aggregate.sh b/benchmarks/site/aggregate.sh new file mode 100644 index 0000000..e1b2dda --- /dev/null +++ b/benchmarks/site/aggregate.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Build the results site's data file from evaluator output and run logs. +# +# The published page renders this JSON and nothing else, so anything that is +# not traceable to a file on disk cannot reach the page. Standardized +# benchmarks only: the retired document suite is not a source here. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +RESULTS_ROOT="${CODE_BENCH_RESULTS_ROOT:-$REPO_ROOT/benchmarks/results/code}" +SUITE="swebench-verified-canary" +OUTPUT="$RESULTS_ROOT/site/leaderboard.json" + +while (( $# > 0 )); do + case "$1" in + --suite) SUITE="${2:?--suite needs a value}"; shift 2 ;; + --output) OUTPUT="${2:?--output needs a value}"; shift 2 ;; + *) printf 'ERROR: unknown aggregate option: %s\n' "$1" >&2; exit 2 ;; + esac +done + +command -v python >/dev/null 2>&1 || { printf 'ERROR: python is required\n' >&2; exit 1; } +[[ -d "$RESULTS_ROOT/evaluation" ]] || { printf 'ERROR: no evaluation directory under %s\n' "$RESULTS_ROOT" >&2; exit 1; } + +python "$SCRIPT_DIR/build-site-data.py" \ + --repo-root "$REPO_ROOT" \ + --results-root "$RESULTS_ROOT" \ + --suite "$SUITE" \ + --output "$OUTPUT" \ + --generated-at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +python "$SCRIPT_DIR/render-page.py" --data "$OUTPUT" --output "${OUTPUT%%.json}.html" diff --git a/benchmarks/site/build-site-data.py b/benchmarks/site/build-site-data.py new file mode 100644 index 0000000..64d4e95 --- /dev/null +++ b/benchmarks/site/build-site-data.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 +"""Build the single JSON the results page renders. + +Every field here is read from a file on disk and carries the path it came +from, so a reader can check any number on the page against the evaluator's +own output. Nothing is entered by hand and nothing is inferred. + +Sources: + benchmarks/code/conditions.json condition ids, labels, tiers + benchmarks/code/suites.json suite and subset pointer + benchmarks/code/subsets/*.json the frozen task list + benchmarks/code/external-sources.lock.json evaluator and dataset pins + results/code/evaluation/*.json official evaluator reports + results/code/evaluation/logs/... per-instance evaluator reports + results/code/runs/*/*// run manifests and provider logs + +Only standardized-benchmark material is read. The retired document suite under +benchmarks/ is not a source here and must never become one. +""" +import argparse +import glob +import json +import os +import re +import subprocess +import sys + +REPORT_NAME_RE = re.compile( + r'^co-evolution-condition-(?P[A-Za-z0-9_-]+)\.(?P.+)\.json$') +GOLD_NAME_RE = re.compile(r'^gold\.(?P.+)\.json$') + + +def read_json(path): + with open(path, encoding='utf-8') as handle: + return json.load(handle) + + +def rel(root, path): + return os.path.relpath(path, root).replace(os.sep, '/') + + +def harness_commit(root): + proc = subprocess.run(('git', '-C', root, 'rev-parse', 'HEAD'), + capture_output=True, text=True) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def harness_dirty(root): + proc = subprocess.run(('git', '-C', root, 'status', '--porcelain'), + capture_output=True, text=True) + return bool(proc.stdout.strip()) if proc.returncode == 0 else None + + +def newest_reports(eval_dir): + """Latest evaluator report per condition, plus the ones it supersedes.""" + latest, superseded = {}, [] + for path in sorted(glob.glob(os.path.join(eval_dir, '*.json'))): + match = REPORT_NAME_RE.match(os.path.basename(path)) + if not match: + continue + cond, run_id = match.group('cond'), match.group('run_id') + previous = latest.get(cond) + if previous is None or run_id > previous[0]: + if previous is not None: + superseded.append((cond, previous[0], previous[1])) + latest[cond] = (run_id, path) + else: + superseded.append((cond, run_id, path)) + return latest, superseded + + +def gold_canary(eval_dir): + best = None + for path in sorted(glob.glob(os.path.join(eval_dir, 'gold.*.json'))): + match = GOLD_NAME_RE.match(os.path.basename(path)) + if not match: + continue + if best is None or match.group('run_id') > best[0]: + best = (match.group('run_id'), path) + if best is None: + return None + data = read_json(best[1]) + return { + 'run_id': best[0], + 'report_file': best[1], + 'submitted': data.get('submitted_instances'), + 'resolved': data.get('resolved_instances'), + } + + +def per_instance_reports(eval_dir, run_id, model_name): + """Read the evaluator's own per-instance verdicts for one scored run.""" + base = os.path.join(eval_dir, 'logs', 'run_evaluation', run_id, model_name) + verdicts = {} + if not os.path.isdir(base): + return verdicts + for instance in sorted(os.listdir(base)): + report = os.path.join(base, instance, 'report.json') + if not os.path.isfile(report): + continue + try: + payload = read_json(report) + except ValueError: + continue + entry = payload.get(instance) or {} + tests = entry.get('tests_status') or {} + f2p = tests.get('FAIL_TO_PASS') or {} + p2p = tests.get('PASS_TO_PASS') or {} + verdicts[instance] = { + 'resolved': bool(entry.get('resolved')), + 'patch_applied': bool(entry.get('patch_successfully_applied')), + 'fail_to_pass_passed': len(f2p.get('success') or []), + 'fail_to_pass_failed': len(f2p.get('failure') or []), + 'pass_to_pass_failed': len(p2p.get('failure') or []), + 'report_file': report, + } + return verdicts + + +def scored_patches(eval_dir, run_id, model_name): + """The exact patch text the evaluator scored, per instance.""" + base = os.path.join(eval_dir, 'logs', 'run_evaluation', run_id, model_name) + patches = {} + if not os.path.isdir(base): + return patches + for instance in sorted(os.listdir(base)): + diff = os.path.join(base, instance, 'patch.diff') + if os.path.isfile(diff): + with open(diff, encoding='utf-8', errors='replace') as handle: + patches[instance] = handle.read() + return patches + + +def index_cells(runs_root): + """Map (condition, instance, patch text) -> cell directory.""" + index = {} + pattern = os.path.join(runs_root, '*', '*', '*', 'prediction.json') + for path in glob.glob(pattern): + try: + record = read_json(path) + except ValueError: + continue + model = record.get('model_name_or_path', '') + if not model.startswith('co-evolution-condition-'): + continue + key = (model[len('co-evolution-condition-'):], + record.get('instance_id'), + (record.get('model_patch') or '').strip()) + index[key] = os.path.dirname(path) + return index + + +def index_attempts(runs_root): + """Cells that ran and recorded an outcome, keyed by (condition, instance). + + A single-shot cell that never produced an applicable patch writes an + outcome.json and no prediction. Without this index such a cell is + indistinguishable from one that was never run, which would read on the page + as missing data rather than as the failure it is. + """ + attempts = {} + for path in glob.glob(os.path.join(runs_root, '*', '*', '*', 'outcome.json')): + try: + record = read_json(path) + except ValueError: + continue + key = (record.get('condition'), record.get('instance')) + attempts[key] = { + 'outcome': record.get('outcome'), + 'attempts': record.get('attempts'), + 'cell_dir': os.path.dirname(path), + } + return attempts + + +def cell_telemetry(cell_dir): + """Provider effort for one cell, from the CLI's own envelope figures.""" + out = { + 'cell_dir': cell_dir, + 'claude_dispatches': 0, + 'claude_cost_usd': 0.0, + 'claude_output_tokens': 0, + 'claude_wall_seconds': 0, + 'codex_phases': 0, + 'single_shot_attempts': None, + 'sandbox': None, + } + manifest = os.path.join(cell_dir, 'run-manifest.json') + if os.path.isfile(manifest): + try: + data = read_json(manifest) + out['sandbox'] = (data.get('sandbox') or {}).get('codex') + out['schema'] = data.get('schema') + except ValueError: + pass + outcome = os.path.join(cell_dir, 'outcome.json') + if os.path.isfile(outcome): + try: + out['single_shot_attempts'] = read_json(outcome).get('attempts') + except ValueError: + pass + logs = os.path.join(cell_dir, 'logs') + if os.path.isdir(logs): + for name in sorted(os.listdir(logs)): + path = os.path.join(logs, name) + if name.startswith('fable-') and name.endswith('.json'): + try: + data = read_json(path) + except ValueError: + continue + out['claude_dispatches'] += 1 + out['claude_cost_usd'] += float(data.get('total_cost_usd') or 0) + usage = data.get('usage') or {} + out['claude_output_tokens'] += int(usage.get('output_tokens') or 0) + out['claude_wall_seconds'] += int((data.get('duration_ms') or 0) / 1000) + elif name.startswith('codex-') and name.endswith('.log'): + out['codex_phases'] += 1 + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--repo-root', required=True) + ap.add_argument('--results-root', required=True) + ap.add_argument('--suite', default='swebench-verified-canary') + ap.add_argument('--output', required=True) + ap.add_argument('--generated-at', required=True, + help='UTC timestamp supplied by the caller') + args = ap.parse_args() + + root = os.path.abspath(args.repo_root) + results = os.path.abspath(args.results_root) + code_dir = os.path.join(root, 'benchmarks', 'code') + eval_dir = os.path.join(results, 'evaluation') + runs_root = os.path.join(results, 'runs') + + conditions = read_json(os.path.join(code_dir, 'conditions.json'))['conditions'] + suites = read_json(os.path.join(code_dir, 'suites.json'))['suites'] + suite = next((s for s in suites if s['id'] == args.suite), None) + if suite is None: + sys.exit('ERROR: unknown suite %s' % args.suite) + subset_path = os.path.join(code_dir, suite['subset_file']) + subset = read_json(subset_path) + instances = [row['instance_id'] for row in subset['instances']] + repos = {row['instance_id']: row['repo'] for row in subset['instances']} + lock = read_json(os.path.join(code_dir, 'external-sources.lock.json')) + + latest, superseded = newest_reports(eval_dir) + cells = index_cells(runs_root) + attempts_index = index_attempts(runs_root) + + rows = [] + for condition in conditions: + cond_id = condition['id'] + row = { + 'condition': cond_id, + 'label': condition['label'], + 'tier': condition['tier'], + 'description': condition['description'], + 'declared_dispatches': condition['dispatches'], + 'measured': False, + 'attempted': False, + 'report_file': None, + 'evaluator_run_id': None, + 'submitted': 0, + 'resolved': 0, + 'per_task': [], + 'telemetry': { + 'claude_dispatches': 0, + 'claude_cost_usd': 0.0, + 'claude_output_tokens': 0, + 'claude_wall_seconds': 0, + 'codex_phases': 0, + 'cells_linked': 0, + 'sandbox_modes': [], + 'single_shot_attempts': [], + }, + } + entry = latest.get(cond_id) + if entry is not None: + run_id, report_path = entry + report = read_json(report_path) + model_name = 'co-evolution-condition-%s' % cond_id + verdicts = per_instance_reports(eval_dir, run_id, model_name) + patches = scored_patches(eval_dir, run_id, model_name) + row['measured'] = True + row['attempted'] = True + row['report_file'] = rel(root, report_path) + row['evaluator_run_id'] = run_id + row['submitted'] = report.get('submitted_instances') or 0 + row['resolved'] = report.get('resolved_instances') or 0 + row['dataset_total_instances'] = report.get('total_instances') + + sandboxes, attempts = set(), [] + for instance in instances: + verdict = verdicts.get(instance) + if verdict is None: + attempt = attempts_index.get((cond_id, instance)) + row['per_task'].append({ + 'instance_id': instance, 'repo': repos[instance], + 'status': 'no-patch' if attempt else 'not-submitted', + 'attempt_outcome': attempt['outcome'] if attempt else None, + 'attempts': attempt['attempts'] if attempt else None, + 'evidence': (rel(root, os.path.join(attempt['cell_dir'], 'outcome.json')) + if attempt else None), + }) + if attempt: + row['attempted'] = True + continue + cell = cells.get((cond_id, instance, + (patches.get(instance) or '').strip())) + task = { + 'instance_id': instance, + 'repo': repos[instance], + 'status': 'resolved' if verdict['resolved'] else 'unresolved', + 'patch_applied': verdict['patch_applied'], + 'fail_to_pass_passed': verdict['fail_to_pass_passed'], + 'fail_to_pass_failed': verdict['fail_to_pass_failed'], + 'pass_to_pass_failed': verdict['pass_to_pass_failed'], + 'evidence': rel(root, verdict['report_file']), + 'cell_dir': rel(root, cell) if cell else None, + } + row['per_task'].append(task) + if cell: + telemetry = cell_telemetry(cell) + row['telemetry']['cells_linked'] += 1 + row['telemetry']['claude_dispatches'] += telemetry['claude_dispatches'] + row['telemetry']['claude_cost_usd'] += telemetry['claude_cost_usd'] + row['telemetry']['claude_output_tokens'] += telemetry['claude_output_tokens'] + row['telemetry']['claude_wall_seconds'] += telemetry['claude_wall_seconds'] + row['telemetry']['codex_phases'] += telemetry['codex_phases'] + if telemetry['sandbox']: + sandboxes.add(telemetry['sandbox']) + if telemetry['single_shot_attempts'] is not None: + attempts.append(telemetry['single_shot_attempts']) + row['telemetry']['sandbox_modes'] = sorted(sandboxes) + row['telemetry']['single_shot_attempts'] = attempts + else: + for instance in instances: + attempt = attempts_index.get((cond_id, instance)) + if attempt: + row['attempted'] = True + row['per_task'].append({ + 'instance_id': instance, 'repo': repos[instance], + 'status': 'no-patch' if attempt else 'not-run', + 'attempt_outcome': attempt['outcome'] if attempt else None, + 'attempts': attempt['attempts'] if attempt else None, + 'evidence': (rel(root, os.path.join(attempt['cell_dir'], 'outcome.json')) + if attempt else None), + }) + row['telemetry']['claude_cost_usd'] = round(row['telemetry']['claude_cost_usd'], 4) + rows.append(row) + + payload = { + 'schema': 'code-bench-site/1.0', + 'generated_at': args.generated_at, + 'harness': { + 'repo_commit': harness_commit(root), + 'working_tree_dirty': harness_dirty(root), + 'swebench_commit': lock.get('swebench', {}).get('commit'), + 'dataset': suite.get('dataset'), + 'dataset_revision': lock.get('dataset', {}).get('revision'), + 'lock_file': rel(root, os.path.join(code_dir, 'external-sources.lock.json')), + }, + 'suite': { + 'id': suite['id'], + 'dataset': suite['dataset'], + 'split': suite['split'], + 'task_count': suite['task_count'], + 'subset_file': rel(root, subset_path), + 'instances': [{'instance_id': i, 'repo': repos[i]} for i in instances], + }, + 'gold_canary': gold_canary(eval_dir), + 'rows': rows, + 'superseded_reports': [ + {'condition': cond, 'evaluator_run_id': run_id, 'report_file': rel(root, path)} + for cond, run_id, path in sorted(superseded) + ], + 'caveat': ('Frozen five-task probe of SWE-bench Verified, scored by the ' + 'official evaluator. One task is 20 points; these numbers are ' + 'not comparable to published full-500 SWE-bench Verified scores.'), + } + + gold = payload['gold_canary'] + if gold: + gold['report_file'] = rel(root, gold['report_file']) + + os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) + with open(args.output, 'w', encoding='utf-8', newline='\n') as handle: + json.dump(payload, handle, indent=2, sort_keys=False) + handle.write('\n') + print(args.output) + + +if __name__ == '__main__': + main() diff --git a/benchmarks/site/render-page.py b/benchmarks/site/render-page.py new file mode 100644 index 0000000..416e87b --- /dev/null +++ b/benchmarks/site/render-page.py @@ -0,0 +1,678 @@ +#!/usr/bin/env python3 +"""Render the results page from the aggregator's JSON and nothing else. + +Every figure on the page is read out of leaderboard.json, which in turn records +the evaluator report or run log each number came from. Prose here is framing; +it never states a result the JSON does not contain. + +The agentic and single-shot tiers are rendered as separate tables on purpose. A +single-shot seat gets one prompt and one answer with no tools and no test run, +so its score is not comparable to a coding agent's and must never appear in the +same ranked list without the label. +""" +import argparse +import html +import json +import os + +TIER_COPY = { + 'agentic': ('Agentic tier', + 'A coding agent works in the repository: reads files, edits them, ' + 'runs tests, and iterates before the patch is taken.'), + 'single-shot': ('Single-shot tier', + 'One prompt, one answer. The model sees the issue and a fixed ' + 'set of retrieved files, returns a diff, and never runs a test ' + 'or looks again. Not comparable to an agentic score.'), +} + +COMPOSITION = { + 'A': 'fable implements', + 'B': 'fable implements → codex repairs', + 'C': 'fable implements → codex + glm + kimi critique → fable repairs', + 'D': 'fable implements → fable reviews and repairs', + 'E': 'codex implements', + 'F': 'glm answers once, from retrieved context', + 'G': 'kimi answers once, from retrieved context', +} + + +def esc(value): + return html.escape('' if value is None else str(value), quote=True) + + +def dots(per_task): + cells = [] + for task in per_task: + status = task['status'] + klass = {'resolved': 'p', 'unresolved': 'f'}.get(status, 'n') + cells.append('' + % (klass, esc(task['instance_id']), esc(status))) + return '
%s
' % ''.join(cells) + + +def score_cell(row, task_count): + if not row['measured']: + # An arm that ran and produced nothing scorable scores zero. Only an arm + # that never ran gets a blank. + if row.get('attempted'): + return ('' + '0%%0 / %d' + '' % task_count) + return 'not run' + # The denominator is the frozen subset, never the number of predictions the + # arm managed to submit. A cell that produced no applicable patch failed its + # task; dropping it from the denominator would flatter the arm. + total = task_count + resolved = row['resolved'] or 0 + pct = (100.0 * resolved / total) if total else 0.0 + remaining = max(0.0, 100.0 - pct) + return ( + '' + '' + '%d%%' + '%d / %d' + ) % (pct, remaining, round(pct), resolved, total) + + +def coverage_chip(row, task_count): + if not row['measured']: + if row.get('attempted'): + return 'ran, no scorable patch' + return 'no data' + linked = row['telemetry']['cells_linked'] + submitted = row['submitted'] or 0 + if submitted < task_count: + return ('%d of %d patches submitted' + % (submitted, task_count)) + if linked < submitted: + return 'telemetry partial' + return 'fully measured' + + +def num_cell(value, sort_value=None, suffix=''): + if value is None: + return '—' + sort_value = value if sort_value is None else sort_value + return '%s%s' % (sort_value, value, suffix) + + +def leaderboard_table(rows, table_id, task_count): + body = [] + for row in rows: + klass = '' if (row['measured'] or row.get('attempted')) else ' class="absent"' + telemetry = row['telemetry'] + claude_calls = telemetry['claude_dispatches'] + seconds = telemetry['claude_wall_seconds'] + cost = telemetry['claude_cost_usd'] + codex_phases = telemetry['codex_phases'] + if row['measured'] or row.get('attempted'): + calls_cell = num_cell(claude_calls if claude_calls else 0) + time_cell = ('%s s' % (seconds, f'{seconds:,}') + if seconds else '—') + cost_cell = ('$%.2f' % (cost, cost) + if cost else 'no CLI figure') + codex_cell = num_cell(codex_phases if codex_phases else 0) + else: + calls_cell = time_cell = cost_cell = codex_cell = '—' + body.append( + '
' + '%s · %s' + '%s
%s%s%s%s%s%s%s' + % (klass, esc(row['condition']), esc(row['label']), + esc(COMPOSITION.get(row['condition'], row['description'])), + score_cell(row, task_count), dots(row['per_task']), + calls_cell, codex_cell, time_cell, cost_cell, coverage_chip(row, task_count))) + return ( + '
' + '' + '' + '' + '' + '' + '' + '' + '' + '%s
PipelineResolvedPer taskFable callsCodex phasesFable timeReported costCoverage
' + ) % (esc(table_id), ''.join(body)) + + +def task_matrix(data): + measured = [row for row in data['rows'] + if row['measured'] or row.get('attempted')] + instances = [entry['instance_id'] for entry in data['suite']['instances']] + header = ''.join('%s%s' + % (esc(row['condition']), + ' single-shot' if row['tier'] == 'single-shot' else '') + for row in measured) + body = [] + for instance in instances: + cells = [] + for row in measured: + task = next((t for t in row['per_task'] if t['instance_id'] == instance), None) + if task is None or task['status'] in ('not-submitted', 'not-run'): + cells.append('not scored') + continue + if task['status'] == 'no-patch': + cells.append('no patch' + % (esc(task.get('attempt_outcome') or 'no applicable patch'), + esc(task.get('attempts')))) + continue + if task['status'] == 'resolved': + cells.append('resolved') + continue + detail = [] + if task.get('fail_to_pass_failed'): + detail.append('%d F2P fail' % task['fail_to_pass_failed']) + if task.get('pass_to_pass_failed'): + detail.append('%d P2P fail' % task['pass_to_pass_failed']) + if not task.get('patch_applied'): + detail.append('patch did not apply') + cells.append('unresolved' + % esc(', '.join(detail) or 'unresolved')) + repo = next(e['repo'] for e in data['suite']['instances'] + if e['instance_id'] == instance) + body.append('%s%s%s' + % (esc(instance), esc(repo), ''.join(cells))) + return ( + '
' + '%s%s
InstanceRepository
' + ) % (header, ''.join(body)) + + +def provenance_table(data): + body = [] + for row in data['rows']: + if not row['measured']: + body.append('%sno evaluator report' + % esc(row['condition'])) + continue + body.append('%s%s%s' + % (esc(row['condition']), esc(row['evaluator_run_id']), esc(row['report_file']))) + for entry in data.get('superseded_reports', []): + body.append('%s%s' + '%s superseded' + % (esc(entry['condition']), esc(entry['evaluator_run_id']), + esc(entry['report_file']))) + return ('
' + '' + '%s
ConditionEvaluator runReport file
') % ''.join(body) + + +def tiles(data): + rows = data['rows'] + measured = [r for r in rows if r['measured'] or r.get('attempted')] + scored_cells = sum(r['submitted'] or 0 for r in measured) + claude_calls = sum(r['telemetry']['claude_dispatches'] for r in measured) + claude_seconds = sum(r['telemetry']['claude_wall_seconds'] for r in measured) + cost = sum(r['telemetry']['claude_cost_usd'] for r in measured) + items = [ + ('Configurations', '%d / %d' % (len(measured), len(rows)), + 'measured on this subset'), + ('Tasks per cell', str(data['suite']['task_count']), 'frozen subset'), + ('Scored cells', str(scored_cells), 'official evaluator, Docker'), + ('Fable dispatches', str(claude_calls), '%s s of model time' % f'{claude_seconds:,}'), + ('Reported cost', '$%.2f' % cost, 'CLI figure, Max plan'), + ] + return '
%s
' % ''.join( + '
%s%s' + '%s
' % (esc(k), v, esc(n)) for k, v, n in items) + + +STYLE = """ + + + + +""" + +SORT_SCRIPT = """ + +""" + + +# The narrative below describes one specific pattern in the results. It is +# guarded: if a later run stops producing that pattern, the section renders +# nothing rather than describing a run that did not happen. +SEPARATOR_TASK = 'sympy__sympy-20916' +UNICODE_DIGITS = ('B', 'D') +ASCII_DIGITS = ('A', 'C', 'E') + + +def status_on(rows, condition, instance): + row = next((r for r in rows if r['condition'] == condition), None) + if row is None: + return None + task = next((t for t in row['per_task'] if t['instance_id'] == instance), None) + return task['status'] if task else None + + +def findings(data): + rows = data['rows'] + resolved_ok = all(status_on(rows, c, SEPARATOR_TASK) == 'resolved' for c in UNICODE_DIGITS) + failed_ok = all(status_on(rows, c, SEPARATOR_TASK) == 'unresolved' for c in ASCII_DIGITS) + others_uniform = True + for entry in data['suite']['instances']: + if entry['instance_id'] == SEPARATOR_TASK: + continue + for cond in UNICODE_DIGITS + ASCII_DIGITS: + if status_on(rows, cond, entry['instance_id']) != 'resolved': + others_uniform = False + if not (resolved_ok and failed_ok and others_uniform): + return '' + + return ( + '

What actually separated the pipelines

' + '

Four of the five tasks resolved under every agentic ' + 'configuration. The entire spread across the leaderboard comes from one task, ' + 'and within that task from one character class.

' + '
' + '
' + 'Finding 01 · ' + esc(SEPARATOR_TASK) + '' + '

Every agentic arm wrote the same fix; two chose Unicode digits and passed

' + '

All five agentic configurations edited the same line of ' + 'sympy/printing/conventions.py, replacing an ASCII-only ' + 'name pattern with a Unicode-aware one. They differ in the second capture group ' + 'alone.

' + '
baseline '
+        "^([a-zA-Z]+)([0-9]+)$\n"
+        'B, D     '
+        "^([^" + chr(92) + "W" + chr(92) + "d_]+)(" + chr(92) + "d+)$     "
+        'Unicode digits → resolved\n'
+        'A, C, E  '
+        "^([^" + chr(92) + "W" + chr(92) + "d_]+)([0-9]+)$  "
+        'ASCII only → unresolved
' + '

The upstream test subscripts a non-ASCII digit, so the ASCII-only class fails it. ' + 'In condition C the panel critique argued explicitly that ' + '' + chr(92) + 'd+ would wrongly capture Arabic-Indic digits; ' + 'the repair pass accepted that reasoning and narrowed the class. The argument was ' + 'careful and the conclusion was wrong.

' + '
' + '
' + 'Finding 02 · harness' + '

The cross-vendor repair arm had never actually run

' + '

Codex on this Windows host accepts ' + '--sandbox workspace-write and then reports ' + 'sandbox: read-only. Across the earlier B cells it wrote a ' + 'review and changed nothing, so that run\'s 5/5 was Fable\'s first draft with a ' + 'discarded review attached.

' + '

The arm now runs with elevated access inside throwaway clones, and the mode is ' + 'recorded per cell. Re-run with a repair step that can write, B resolves the separator ' + 'task. Its two earlier reports are listed as superseded below rather than deleted.

' + '
' + '
') + + +def build(data): + harness = data['harness'] + gold = data['gold_canary'] or {} + suite = data['suite'] + rows = data['rows'] + agentic = [r for r in rows if r['tier'] == 'agentic'] + single = [r for r in rows if r['tier'] == 'single-shot'] + measured_single = [r for r in single if r['measured'] or r.get('attempted')] + + parts = [] + parts.append('Co-Evolution Code Battery') + parts.append(STYLE) + parts.append('
') + + parts.append( + '
' + '
SWE-bench Verified · %d-task frozen subset' + 'Official pinned evaluator, Docker' + 'Built %s
' + '

Co-Evolution Code Battery

' + '

Does putting a second model in the loop produce better patches ' + 'than one model working alone? Seven configurations, five pinned SWE-bench Verified ' + 'tasks, every patch scored by the official evaluator in Docker.

' + '
fable @ mediumgpt-5.6-sol @ medium' + 'glm-5.3-flash @ effort:lowkimi-k3 @ thinking:off' + 'phase timeout 900sgold canary %s/%s' + 'harness %s
' + % (suite['task_count'], esc(data['generated_at']), + esc(gold.get('resolved')), esc(gold.get('submitted')), + esc((harness.get('repo_commit') or '')[:7]))) + + parts.append( + '

Read this before the table

' + '

Five tasks is a probe, not a ranking. One task is 20 points, so a one-task gap ' + 'between two rows is well inside what a five-task sample produces by chance. ' + 'These numbers are also not comparable to published full-500 SWE-bench Verified ' + 'scores: the subset is fixed and was chosen for the harness, not drawn at random.

' + '

The two tiers are listed separately because they are not the same test. An agentic ' + 'row had file tools and could run the test suite; a single-shot row got one prompt and ' + 'answered once.

') + + parts.append(tiles(data)) + + parts.append('

Leaderboard

') + for tier, group in (('agentic', agentic), ('single-shot', single)): + title, note = TIER_COPY[tier] + parts.append('

%s

%s

' % (esc(title), esc(note))) + parts.append(leaderboard_table(group, 'board-%s' % tier, suite['task_count'])) + parts.append( + '
' + ' resolved' + ' unresolved' + ' no patch scored' + 'Task order: %s
' + % esc(' · '.join(e['instance_id'].split('__')[0] for e in suite['instances']))) + parts.append('
') + + parts.append(findings(data)) + + parts.append( + '

Task by task

' + '

One column per measured configuration. Hover an unresolved cell ' + 'for the failing test counts the evaluator recorded.

%s
' % task_matrix(data)) + + integrity = [] + integrity.append( + '
  • Evaluator. Official SWE-bench harness pinned at ' + '%s, dataset %s at revision ' + '%s, both from %s.
    • ' + % (esc((harness.get('swebench_commit') or '')[:12]), esc(harness.get('dataset')), + esc((harness.get('dataset_revision') or '')[:12]), esc(harness.get('lock_file')))) + if gold: + integrity.append( + '
  • Gold canary. A gold patch resolved %s/%s before any generated ' + 'prediction was scored, so a failure here means the patch, not the harness. Report: ' + '%s.
    • ' + % (esc(gold.get('resolved')), esc(gold.get('submitted')), esc(gold.get('report_file')))) + integrity.append( + '
  • Cost is the CLI\'s own figure for work billed to a Max ' + 'subscription, not metered API spend. The Claude CLI exposes no plan meter, so there is ' + 'no percentage-of-plan number here. Codex, GLM and Kimi report no per-call cost, so their ' + 'rows show dispatch counts instead.
    • ') + if measured_single: + integrity.append( + '
  • The single-shot tier is a floor, not a model ceiling. GLM and ' + 'Kimi are reachable here only as chat completions. Their cells receive the issue plus ' + 'a deterministic file selection and return one diff, gated by ' + 'git apply --check --recount. A cell that never produced an ' + 'applicable patch contributes no prediction and still counts against the five tasks. ' + '--recount recomputes the hunk line counts and changes no ' + 'line of the proposed edit: without it the gate scores a chat model\'s ability to ' + 'count lines, which the agentic arms never have to do because they edit files ' + 'directly.
    • ') + by_mode = {} + for row in rows: + for mode in row['telemetry']['sandbox_modes']: + by_mode.setdefault(mode, []).append(row['condition']) + if by_mode: + detail = '; '.join('%s in %s' + % (esc(mode), esc(', '.join(sorted(conds)))) + for mode, conds in sorted(by_mode.items())) + integrity.append( + '
  • Codex sandbox. %s. Codex on Windows accepts ' + 'workspace-write and then runs read-only, which makes a ' + 'repair arm look like it ran while changing nothing, so the mode each cell used is ' + 'recorded in its run manifest. The arms that needed Codex to write ran with elevated ' + 'access inside throwaway clones.
    • ' % detail) + integrity.append( + '
  • Every number on this page comes from a file. The page is generated ' + 'from benchmarks/site/aggregate.sh output; each row names the ' + 'evaluator report it was read from and each task cell is backed by that run\'s per-instance ' + 'report.json.
    • ') + + parts.append('

    Methodology and integrity

    ' + '
      %s
    %s
    ' + % (''.join(integrity), provenance_table(data))) + + dirty = harness.get('working_tree_dirty') + parts.append( + '
    Generated from run artifacts in ' + 'benchmarks/results/code/ by ' + 'benchmarks/site/aggregate.sh.
    ' + '
    Repository co-evolution-runtime · harness commit %s%s
    ' + '
    %s
    ' + % (esc(harness.get('repo_commit')), + ' · working tree had uncommitted changes at build time' if dirty else '', + esc(data['caveat']))) + + parts.append('
    ') + parts.append(SORT_SCRIPT) + return '\n'.join(parts) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--data', required=True) + ap.add_argument('--output', required=True) + args = ap.parse_args() + with open(args.data, encoding='utf-8') as handle: + data = json.load(handle) + page = build(data) + page = page.replace(' Date: Wed, 2 Sep 2026 15:58:53 -0400 Subject: [PATCH 10/10] Add 50-task SWE-bench Verified suite definition The hand-pinned 5-task canary is one instance per repository, so its score cannot be extrapolated to the full split. Add a uniform random 50-task draw (seed recorded in the subset file) and make the suite selectable via CODE_BENCH_SUITE so existing canary invocations keep working. Also scrub cloned workspace history down to a single synthetic base commit: a full clone carries the upstream fix for the very issue under test, and observed runs read the answer out of `git log` instead of deriving it. Co-Authored-By: Claude Opus 5 --- benchmarks/code/code-bench.sh | 11 +- benchmarks/code/lib/code-bench-lib.sh | 19 +- benchmarks/code/scripts/draw-subset.py | 97 ++++++++ benchmarks/code/scripts/evaluate-swebench.sh | 2 +- .../code/scripts/prepare-swebench-instance.sh | 23 +- .../subsets/swebench-verified-random50.json | 217 ++++++++++++++++++ benchmarks/code/suites.json | 21 +- 7 files changed, 381 insertions(+), 9 deletions(-) create mode 100644 benchmarks/code/scripts/draw-subset.py create mode 100644 benchmarks/code/subsets/swebench-verified-random50.json diff --git a/benchmarks/code/code-bench.sh b/benchmarks/code/code-bench.sh index 4f608db..2370c25 100644 --- a/benchmarks/code/code-bench.sh +++ b/benchmarks/code/code-bench.sh @@ -16,12 +16,16 @@ case "$COMMAND" in exec bash "$SCRIPT_DIR/estimate-compute.sh" "$@" ;; fetch-metadata) - suite_json=$(code_suite_json "swebench-verified-canary") + suite_json=$(code_suite_json "$(code_suite_id)") subset=$(code_subset_path "$suite_json") - output="$CODE_BENCH_RESULTS_ROOT/metadata/swebench-verified-canary.json" + output=$(code_metadata_path) exec python "$SCRIPT_DIR/scripts/fetch-swebench-metadata.py" \ --subset "$subset" --lock "$SCRIPT_DIR/external-sources.lock.json" --output "$output" ;; + draw-subset) + exec python "$SCRIPT_DIR/scripts/draw-subset.py" \ + --lock "$SCRIPT_DIR/external-sources.lock.json" "$@" + ;; setup) exec bash "$SCRIPT_DIR/scripts/setup-swebench.sh" "$@" ;; @@ -52,7 +56,8 @@ usage: code-bench.sh COMMAND [options] check validate checked-in manifests estimate [options] report declared provider dispatches - fetch-metadata cache public inputs for the frozen subset + fetch-metadata cache public inputs for the active suite + draw-subset [options] draw a reproducible random subset setup --check|--install inspect or install pinned SWE-bench tooling prepare-instance ID RUN COND clone a clean public-input workspace run-workflow [options] generate one capped condition prediction diff --git a/benchmarks/code/lib/code-bench-lib.sh b/benchmarks/code/lib/code-bench-lib.sh index cc2520c..c3f89cc 100644 --- a/benchmarks/code/lib/code-bench-lib.sh +++ b/benchmarks/code/lib/code-bench-lib.sh @@ -13,6 +13,16 @@ code_require() { command -v "$1" >/dev/null 2>&1 || code_die "$1 is required" } +# Which suite the commands operate on. The canary stays the default so every +# existing invocation keeps working; a larger suite is opt-in per command. +code_suite_id() { + printf '%s' "${CODE_BENCH_SUITE:-swebench-verified-canary}" +} + +code_metadata_path() { + printf '%s/metadata/%s.json' "$CODE_BENCH_RESULTS_ROOT" "$(code_suite_id)" +} + code_suite_json() { local suite="$1" jq -ce --arg id "$suite" '.suites[] | select(.id == $id)' "$CODE_BENCH_DIR/suites.json" @@ -68,11 +78,16 @@ code_check_manifests() { fi if ! jq -e '.schema == "code-bench-subset/1.0" and (.instances | length > 0) and - ([.instances[].instance_id] | length == (unique | length)) and - ([.instances[].repo] | length == (unique | length))' "$subset" >/dev/null; then + ([.instances[].instance_id] | length == (unique | length))' "$subset" >/dev/null; then printf 'CHECK FAIL: malformed or duplicate subset entries: %s\n' "$subset" >&2 failures=$((failures + 1)); continue fi + if printf '%s' "$suite_json" | jq -e '.require_unique_repos == true' >/dev/null 2>&1; then + if ! jq -e '[.instances[].repo] | length == (unique | length)' "$subset" >/dev/null; then + printf 'CHECK FAIL: suite requires one task per repository: %s\n' "$subset" >&2 + failures=$((failures + 1)); continue + fi + fi count=$(jq '.instances | length' "$subset" | tr -d '\r') if [[ "$count" != "$declared" ]]; then printf 'CHECK FAIL: suite declares %s tasks but subset contains %s: %s\n' "$declared" "$count" "$subset" >&2 diff --git a/benchmarks/code/scripts/draw-subset.py b/benchmarks/code/scripts/draw-subset.py new file mode 100644 index 0000000..42568f2 --- /dev/null +++ b/benchmarks/code/scripts/draw-subset.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Draw a reproducible random subset of a SWE-bench split. + +The existing canary subset was hand-pinned, one instance per repository, which +makes it useful for harness development and useless for estimating a score: a +non-random subset cannot be extrapolated to the full split at any sample size. +This draws a uniform random sample instead, from the same revision the lock file +pins, so the result is an unbiased estimate of the full-split score. + +The seed is written into the subset file. Re-running with the same seed and the +same dataset revision reproduces the identical sample. +""" +from __future__ import annotations + +import argparse +import json +import random +import urllib.parse +import urllib.request +from pathlib import Path + + +def get_json(url: str) -> dict: + with urllib.request.urlopen(url, timeout=60) as response: + return json.load(response) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--lock", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--suite-id", required=True) + parser.add_argument("--size", type=int, required=True) + parser.add_argument("--seed", type=int, required=True) + parser.add_argument("--split", default="test") + parser.add_argument("--total", type=int, default=500) + args = parser.parse_args() + + lock = json.loads(args.lock.read_text(encoding="utf-8")) + dataset_id = lock["dataset"]["id"] + expected = lock["dataset"]["revision"] + actual = get_json(f"https://huggingface.co/api/datasets/{dataset_id}").get("sha") + if actual != expected: + raise SystemExit(f"dataset revision drift: expected {expected}, got {actual}") + + api_id = urllib.parse.quote(dataset_id, safe="") + population: list[dict] = [] + for offset in range(0, args.total, 100): + url = ( + "https://datasets-server.huggingface.co/rows" + f"?dataset={api_id}&config=default&split={args.split}" + f"&offset={offset}&length=100" + ) + for wrapped in get_json(url).get("rows", []): + row = wrapped["row"] + population.append({ + "instance_id": row["instance_id"], + "repo": row["repo"], + }) + + if len(population) != args.total: + raise SystemExit(f"expected {args.total} instances, enumerated {len(population)}") + if args.size > len(population): + raise SystemExit("sample size exceeds the split") + + # Sort first so the draw depends only on the seed, never on server row order. + population.sort(key=lambda r: r["instance_id"]) + sample = random.Random(args.seed).sample(population, args.size) + sample.sort(key=lambda r: r["instance_id"]) + + payload = { + "schema": "code-bench-subset/1.0", + "suite_id": args.suite_id, + "sampling": { + "method": "uniform-random-without-replacement", + "seed": args.seed, + "drawn_from": {"dataset": dataset_id, "revision": expected, + "split": args.split, "population": len(population)}, + "note": ("A uniform random draw, so the observed score is an unbiased " + "estimate of the full-split score with a binomial interval."), + }, + "instances": sample, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + repos: dict[str, int] = {} + for row in sample: + repos[row["repo"]] = repos.get(row["repo"], 0) + 1 + print(f"WROTE: {len(sample)} of {len(population)} -> {args.output}") + for repo, count in sorted(repos.items(), key=lambda kv: (-kv[1], kv[0])): + print(f" {count:3d} {repo}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/code/scripts/evaluate-swebench.sh b/benchmarks/code/scripts/evaluate-swebench.sh index f55c036..efd29eb 100644 --- a/benchmarks/code/scripts/evaluate-swebench.sh +++ b/benchmarks/code/scripts/evaluate-swebench.sh @@ -8,7 +8,7 @@ source "$CODE_DIR/lib/code-bench-lib.sh" MODE="${1:-}" shift || true -SUITE="swebench-verified-canary" +SUITE=$(code_suite_id) CACHE="$CODE_BENCH_RESULTS_ROOT/.cache" if [[ -x "$CACHE/venv/Scripts/swebench.exe" ]]; then CLI="$CACHE/venv/Scripts/swebench.exe"; else CLI="$CACHE/venv/bin/swebench"; fi diff --git a/benchmarks/code/scripts/prepare-swebench-instance.sh b/benchmarks/code/scripts/prepare-swebench-instance.sh index e4ebf7d..197286e 100644 --- a/benchmarks/code/scripts/prepare-swebench-instance.sh +++ b/benchmarks/code/scripts/prepare-swebench-instance.sh @@ -17,10 +17,10 @@ CONDITION="${3:-}" jq -e --arg id "$CONDITION" 'any(.conditions[]; .id == $id)' "$CODE_DIR/conditions.json" >/dev/null \ || { code_die "unknown condition: $CONDITION"; exit 2; } -METADATA="$CODE_BENCH_RESULTS_ROOT/metadata/swebench-verified-canary.json" +METADATA=$(code_metadata_path) [[ -f "$METADATA" ]] || { code_die "public metadata is absent; run code-bench.sh fetch-metadata"; exit 1; } row=$(jq -ce --arg id "$INSTANCE" '.instances[] | select(.instance_id == $id)' "$METADATA") \ - || { code_die "instance is outside the frozen canary: $INSTANCE"; exit 1; } + || { code_die "instance is outside suite $(code_suite_id): $INSTANCE"; exit 1; } repo=$(printf '%s' "$row" | jq -r '.repo' | tr -d '\r') base_commit=$(printf '%s' "$row" | jq -r '.base_commit' | tr -d '\r') [[ "$repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || { code_die "unsafe repository id: $repo"; exit 1; } @@ -36,6 +36,25 @@ mkdir -p "$CELL" git clone --filter=blob:none --no-checkout "https://github.com/$repo.git" "$WORKSPACE" git -C "$WORKSPACE" checkout --detach "$base_commit" + +# A clone carries every commit AFTER the base, including the upstream fix for +# this very issue. An agent that runs `git log origin/main` can read the answer +# instead of deriving it, and observed runs did exactly that -- citing the +# upstream PR and commit hash back in their reports. Replace the history with a +# single synthetic commit holding the base tree, so the working tree is +# identical but nothing about the future is recoverable. `git diff` still yields +# the agent's changes, which is all the driver needs. +rm -rf "$WORKSPACE/.git" +git -C "$WORKSPACE" init -q +git -C "$WORKSPACE" -c core.autocrlf=false add -A +git -C "$WORKSPACE" -c user.email=bench@local -c user.name=bench \ + commit -q -m "base $base_commit" --no-gpg-sign +if git -C "$WORKSPACE" log --oneline --all | wc -l | grep -qv '^ *1$'; then + code_die "workspace history was not reduced to a single commit"; exit 1 +fi +if git -C "$WORKSPACE" remote -v | grep -q .; then + code_die "workspace still has a remote configured"; exit 1 +fi git -C "$WORKSPACE" status --porcelain | grep -q . && { code_die "prepared workspace is unexpectedly dirty: $WORKSPACE"; exit 1; } diff --git a/benchmarks/code/subsets/swebench-verified-random50.json b/benchmarks/code/subsets/swebench-verified-random50.json new file mode 100644 index 0000000..5ac29ca --- /dev/null +++ b/benchmarks/code/subsets/swebench-verified-random50.json @@ -0,0 +1,217 @@ +{ + "schema": "code-bench-subset/1.0", + "suite_id": "swebench-verified-random50", + "sampling": { + "method": "uniform-random-without-replacement", + "seed": 20260901, + "drawn_from": { + "dataset": "princeton-nlp/SWE-bench_Verified", + "revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a", + "split": "test", + "population": 500 + }, + "note": "A uniform random draw, so the observed score is an unbiased estimate of the full-split score with a binomial interval." + }, + "instances": [ + { + "instance_id": "astropy__astropy-12907", + "repo": "astropy/astropy" + }, + { + "instance_id": "astropy__astropy-13033", + "repo": "astropy/astropy" + }, + { + "instance_id": "astropy__astropy-13398", + "repo": "astropy/astropy" + }, + { + "instance_id": "astropy__astropy-14365", + "repo": "astropy/astropy" + }, + { + "instance_id": "django__django-11163", + "repo": "django/django" + }, + { + "instance_id": "django__django-11179", + "repo": "django/django" + }, + { + "instance_id": "django__django-11451", + "repo": "django/django" + }, + { + "instance_id": "django__django-11477", + "repo": "django/django" + }, + { + "instance_id": "django__django-12754", + "repo": "django/django" + }, + { + "instance_id": "django__django-13128", + "repo": "django/django" + }, + { + "instance_id": "django__django-13807", + "repo": "django/django" + }, + { + "instance_id": "django__django-14053", + "repo": "django/django" + }, + { + "instance_id": "django__django-14315", + "repo": "django/django" + }, + { + "instance_id": "django__django-14534", + "repo": "django/django" + }, + { + "instance_id": "django__django-14631", + "repo": "django/django" + }, + { + "instance_id": "django__django-14792", + "repo": "django/django" + }, + { + "instance_id": "django__django-15022", + "repo": "django/django" + }, + { + "instance_id": "django__django-15280", + "repo": "django/django" + }, + { + "instance_id": "django__django-15380", + "repo": "django/django" + }, + { + "instance_id": "django__django-15525", + "repo": "django/django" + }, + { + "instance_id": "django__django-15731", + "repo": "django/django" + }, + { + "instance_id": "django__django-16100", + "repo": "django/django" + }, + { + "instance_id": "django__django-16263", + "repo": "django/django" + }, + { + "instance_id": "django__django-16493", + "repo": "django/django" + }, + { + "instance_id": "django__django-16612", + "repo": "django/django" + }, + { + "instance_id": "django__django-16819", + "repo": "django/django" + }, + { + "instance_id": "matplotlib__matplotlib-21568", + "repo": "matplotlib/matplotlib" + }, + { + "instance_id": "matplotlib__matplotlib-24970", + "repo": "matplotlib/matplotlib" + }, + { + "instance_id": "mwaskom__seaborn-3069", + "repo": "mwaskom/seaborn" + }, + { + "instance_id": "psf__requests-6028", + "repo": "psf/requests" + }, + { + "instance_id": "pydata__xarray-4075", + "repo": "pydata/xarray" + }, + { + "instance_id": "pydata__xarray-4687", + "repo": "pydata/xarray" + }, + { + "instance_id": "pydata__xarray-6599", + "repo": "pydata/xarray" + }, + { + "instance_id": "pydata__xarray-6721", + "repo": "pydata/xarray" + }, + { + "instance_id": "pydata__xarray-6992", + "repo": "pydata/xarray" + }, + { + "instance_id": "pytest-dev__pytest-7571", + "repo": "pytest-dev/pytest" + }, + { + "instance_id": "scikit-learn__scikit-learn-12682", + "repo": "scikit-learn/scikit-learn" + }, + { + "instance_id": "scikit-learn__scikit-learn-13142", + "repo": "scikit-learn/scikit-learn" + }, + { + "instance_id": "scikit-learn__scikit-learn-13496", + "repo": "scikit-learn/scikit-learn" + }, + { + "instance_id": "scikit-learn__scikit-learn-14983", + "repo": "scikit-learn/scikit-learn" + }, + { + "instance_id": "sphinx-doc__sphinx-10614", + "repo": "sphinx-doc/sphinx" + }, + { + "instance_id": "sphinx-doc__sphinx-8120", + "repo": "sphinx-doc/sphinx" + }, + { + "instance_id": "sphinx-doc__sphinx-8269", + "repo": "sphinx-doc/sphinx" + }, + { + "instance_id": "sphinx-doc__sphinx-8621", + "repo": "sphinx-doc/sphinx" + }, + { + "instance_id": "sphinx-doc__sphinx-8638", + "repo": "sphinx-doc/sphinx" + }, + { + "instance_id": "sympy__sympy-12419", + "repo": "sympy/sympy" + }, + { + "instance_id": "sympy__sympy-12489", + "repo": "sympy/sympy" + }, + { + "instance_id": "sympy__sympy-13480", + "repo": "sympy/sympy" + }, + { + "instance_id": "sympy__sympy-16450", + "repo": "sympy/sympy" + }, + { + "instance_id": "sympy__sympy-23262", + "repo": "sympy/sympy" + } + ] +} diff --git a/benchmarks/code/suites.json b/benchmarks/code/suites.json index 23ef5db..5e258ee 100644 --- a/benchmarks/code/suites.json +++ b/benchmarks/code/suites.json @@ -7,7 +7,26 @@ "split": "test", "subset_file": "subsets/swebench-verified-canary.json", "task_count": 5, - "default_conditions": ["A", "B", "C", "D", "E", "F", "G"] + "default_conditions": [ + "A", + "B", + "C", + "D", + "E", + "F", + "G" + ], + "require_unique_repos": true + }, + { + "id": "swebench-verified-random50", + "dataset": "princeton-nlp/SWE-bench_Verified", + "split": "test", + "subset_file": "subsets/swebench-verified-random50.json", + "task_count": 50, + "default_conditions": [ + "B" + ] } ] }