diff --git a/Makefile b/Makefile index 1678d48..939030e 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ endif smoke-run smoke-dry resume postmortem \ eval-core-test litellm-up litellm-down stack-up stack-down stack-status \ openrouter-smoke env-check \ - harness-image-aider sandbox-net-up + harness-image-aider harness-image-goose sandbox-net-up demo-run: python -m pollmevals_eval_core.demo_run --tasks evals/tasks --output artifacts @@ -118,6 +118,11 @@ stack-status: harness-image-aider: docker build -t pollmevals-harness-aider:0.1.0 infra/docker/harness-aider/ +# Build the goose harness image (RFC-006 Phase 5). No spend — pulls base + +# the Block goose release binary. goose is model-agnostic (OpenAI-compatible). +harness-image-goose: + docker build -t pollmevals-harness-goose:0.1.0 infra/docker/harness-goose/ + # Build the Python eval sandbox image (Phase 5 — PythonCorrectnessEvaluator, # runs BigCodeBench unittest suites). No spend — pulls base + pip scientific stack. eval-image-py: diff --git a/apps/eval-core-py/scripts/build_real_board.py b/apps/eval-core-py/scripts/build_real_board.py index dc1ec85..6f57440 100644 --- a/apps/eval-core-py/scripts/build_real_board.py +++ b/apps/eval-core-py/scripts/build_real_board.py @@ -83,6 +83,12 @@ "grok-4", ] _AIDER_MODELS = ["qwen-3-14b", "qwen3-coder-30b", "codestral", "devstral"] +# goose runs the SAME coder models as aider, so each (model) gives a directly +# comparable pair — isolating the harness variable (aider L4 vs goose L2 on +# identical models). Diverge this list later if goose handles models aider can't. +_GOOSE_MODELS = ["qwen-3-14b", "qwen3-coder-30b", "codestral", "devstral"] +# Per-stack candidate model lists for --add-stack (merge ONE harness column in). +_STACK_MODELS = {"aider": _AIDER_MODELS, "goose": _GOOSE_MODELS} _SEEDS = [1, 2] _TASK = "be_01_jwt_auth" _JUDGES = ["claude-sonnet-4-6-judge", "gpt-5-mini-judge", "gemini-3-flash"] @@ -171,6 +177,14 @@ async def _main() -> int: "existing board.json (fills previously-failed cells without re-running " "the rest). e.g. --fill grok-4", ) + ap.add_argument( + "--add-stack", + default="", + help="run ONLY this stack's grid (on _STACK_MODELS[stack]) and MERGE its " + "new harness column (cells + harness metadata) into the existing " + "board.json, without re-spending on the other harnesses. e.g. " + "--add-stack goose", + ) args = ap.parse_args() os.chdir(REPO) @@ -275,6 +289,53 @@ def caller_for(stack_id: str) -> object: print(f" {c.stack_id} x {c.model_id}: score={c.mean_score} cost=${c.mean_cost_usd}") return 0 + # --add-stack: run ONLY this harness's grid and merge its NEW column (cells + + # harness metadata) into the existing board.json, without re-spending on the + # other harnesses. Unlike --fill (replace cells in place), this also unions + # the new harness into board.harnesses so the matrix renders the column. + if args.add_stack: + stack_id = args.add_stack + models = _STACK_MODELS.get(stack_id, _AIDER_MODELS) + print(f"ADD-STACK: {stack_id} x {models} → merge column into {out.name} ...") + result = await runner.run( + GridSpec( + run_hash=_RUN_HASH, + models=models, + tasks=[_TASK], + stacks=[stack_id], + seeds=_SEEDS, + task_timeout_s=task_timeout, + ) + ) + rows = _rows_from_result(result, _PRICING) + partial = build_board(rows, stacks_root=stacks_root, run_hash=_RUN_HASH, run_type="smoke") + existing = Board.model_validate_json(out.read_text(encoding="utf-8")) + # Replace-or-append cells by (model, stack); union harnesses + models. + by_key = {(c.model_id, c.stack_id): c for c in existing.cells} + for c in partial.cells: + by_key[(c.model_id, c.stack_id)] = c + h_by_id = {h.stack_id: h for h in existing.harnesses} + for h in partial.harnesses: + h_by_id[h.stack_id] = h + m_by_id = {m.model_id: m for m in existing.models} + for m in partial.models: + m_by_id.setdefault(m.model_id, m) + cells = list(by_key.values()) + scored_now = sum(1 for c in cells if c.mean_score is not None) + merged_board = existing.model_copy( + update={ + "cells": cells, + "harnesses": list(h_by_id.values()), + "models": list(m_by_id.values()), + "scored": scored_now > 0, + } + ) + out.write_text(merged_board.model_dump_json(indent=2) + "\n", encoding="utf-8") + print(f" board now {scored_now}/{len(cells)} cells scored. new {stack_id} cells:") + for c in partial.cells: + print(f" {c.stack_id} x {c.model_id}: score={c.mean_score} cost=${c.mean_cost_usd}") + return 0 + # Two specs so the grid is non-cartesian: raw-llm on every candidate, aider # only on the models that follow its edit format. Same run_hash + runner so # rows merge into one board. diff --git a/apps/eval-core-py/src/orchestrator/stack_executor.py b/apps/eval-core-py/src/orchestrator/stack_executor.py index f637498..c52c6f8 100644 --- a/apps/eval-core-py/src/orchestrator/stack_executor.py +++ b/apps/eval-core-py/src/orchestrator/stack_executor.py @@ -231,10 +231,45 @@ def _aider_invocation( ) +def _goose_invocation( + proxy_base_url: str, api_key: str, model_alias: str, prompt: str +) -> ProxyInvocation: + """Goose recipe — PROVEN (2026-06-03 isolation smoke; memory). + + Block's goose is model-agnostic via an OpenAI-compatible provider. Unlike + aider's single ``OPENAI_API_BASE``, goose splits the endpoint into + ``OPENAI_HOST`` (scheme+host, NO path) + ``OPENAI_BASE_PATH`` (the chat + route), and selects provider/model via ``GOOSE_PROVIDER`` / ``GOOSE_MODEL``. + ``GOOSE_DISABLE_KEYRING`` (no system keyring in a sandbox container) and + ``GOOSE_MODE=auto`` (never block waiting on a tool-call confirmation) are + baked into the image; set here too so the recipe is self-contained. The + model is chosen by env, so ``extra_args`` is empty; the prompt rides ``-t`` + and the ``goose run --no-session --with-builtin developer`` scaffolding + lives in stack.yaml ``execution.args``. + """ + base = proxy_base_url.rstrip("/") + return ProxyInvocation( + env={ + "GOOSE_PROVIDER": "openai", + "GOOSE_MODEL": model_alias, + "GOOSE_MODE": "auto", + "GOOSE_DISABLE_KEYRING": "1", + "OPENAI_API_KEY": api_key, + "OPENAI_HOST": base, + "OPENAI_BASE_PATH": "v1/chat/completions", + }, + config_files={}, + extra_args=[], + prompt_args=["-t", prompt], + ) + + # Proven recipes (validated end-to-end via the proxy). aider is the RFC-006 -# first slice (aider x qwen x be_01). +# first slice (aider x qwen x be_01); goose is the second harness (2026-06-03), +# a model-agnostic peer that runs the same coder models for a clean comparison. _PROVEN_RECIPES: dict[str, _RecipeBuilder] = { "aider": _aider_invocation, + "goose": _goose_invocation, } # Known harnesses whose recipe is proven in spikes but lands at its per-stack @@ -244,7 +279,6 @@ def _aider_invocation( "claude-code", "codex", "opencode", - "goose", "openhands", "hermes", "cline", diff --git a/apps/eval-core-py/tests/test_stack_executor.py b/apps/eval-core-py/tests/test_stack_executor.py index 6a6dab8..c7bf722 100644 --- a/apps/eval-core-py/tests/test_stack_executor.py +++ b/apps/eval-core-py/tests/test_stack_executor.py @@ -188,7 +188,36 @@ def test_aider_strips_trailing_slash(self) -> None: ) assert inv.env["OPENAI_API_BASE"] == "http://localhost:4000/v1" - @pytest.mark.parametrize("cli", ["claude-code", "codex", "goose", "openhands", "opencode"]) + def test_goose_recipe_is_proven(self) -> None: + inv = build_proxy_invocation( + "goose", + proxy_base_url="http://pollmevals-litellm-proxy:4000", + api_key="sk-local-xyz", + model_alias="qwen-3-14b", + prompt="do the thing", + ) + assert inv.env["GOOSE_PROVIDER"] == "openai" + assert inv.env["GOOSE_MODEL"] == "qwen-3-14b" + assert inv.env["OPENAI_API_KEY"] == "sk-local-xyz" + # goose splits the endpoint: host (NO path) + the chat route separately. + assert inv.env["OPENAI_HOST"] == "http://pollmevals-litellm-proxy:4000" + assert inv.env["OPENAI_BASE_PATH"] == "v1/chat/completions" + assert inv.env["GOOSE_DISABLE_KEYRING"] == "1" + assert inv.extra_args == [] # model selected by env, not a CLI flag + assert inv.prompt_args == ["-t", "do the thing"] + assert inv.config_files == {} + + def test_goose_strips_trailing_slash(self) -> None: + inv = build_proxy_invocation( + "goose", + proxy_base_url="http://h:4000/", + api_key="k", + model_alias="m", + prompt="p", + ) + assert inv.env["OPENAI_HOST"] == "http://h:4000" + + @pytest.mark.parametrize("cli", ["claude-code", "codex", "openhands", "opencode"]) def test_known_but_pending_harness_raises_pending(self, cli: str) -> None: with pytest.raises(HarnessRecipePending, match="Phase 5"): build_proxy_invocation( @@ -207,8 +236,8 @@ def test_none_cli_raises_unsupported(self) -> None: None, proxy_base_url="x", api_key="k", model_alias="m", prompt="p" ) - def test_supported_harnesses_is_aider_only_in_phase_1(self) -> None: - assert supported_harnesses() == frozenset({"aider"}) + def test_supported_harnesses_are_aider_and_goose(self) -> None: + assert supported_harnesses() == frozenset({"aider", "goose"}) # --------------------------------------------------------------------------- diff --git a/apps/site/public/board.json b/apps/site/public/board.json index 5dbd42c..1bfda81 100644 --- a/apps/site/public/board.json +++ b/apps/site/public/board.json @@ -26,6 +26,16 @@ "L0_bare_llm" ], "family": "baseline" + }, + { + "stack_id": "goose", + "name": "Goose", + "level": 2, + "layers": [ + "L1_system_prompt", + "L2_tools" + ], + "family": "agnostic" } ], "models": [ @@ -629,6 +639,99 @@ "type_safety": 5.25 }, "on_frontier": false + }, + { + "model_id": "codestral", + "stack_id": "goose", + "mean_score": null, + "mean_cost_usd": 0.0, + "mean_latency_ms": 9353, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": null, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": {}, + "on_frontier": false + }, + { + "model_id": "devstral", + "stack_id": "goose", + "mean_score": 7.12, + "mean_cost_usd": 0.0, + "mean_latency_ms": 200855, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 7.12, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 7.5, + "correctness": 7.75, + "error_handling": 7.0, + "security_posture": 8.0, + "test_alignment": 7.0, + "type_safety": 5.5 + }, + "on_frontier": false + }, + { + "model_id": "qwen-3-14b", + "stack_id": "goose", + "mean_score": 6.0, + "mean_cost_usd": 0.0, + "mean_latency_ms": 59121, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 6.0, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 7.0, + "correctness": 4.5, + "error_handling": 6.0, + "security_posture": 6.5, + "test_alignment": 7.0, + "type_safety": 5.0 + }, + "on_frontier": false + }, + { + "model_id": "qwen3-coder-30b", + "stack_id": "goose", + "mean_score": 7.17, + "mean_cost_usd": 0.0, + "mean_latency_ms": 48202, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 7.17, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 7.5, + "correctness": 8.0, + "error_handling": 7.5, + "security_posture": 8.25, + "test_alignment": 6.5, + "type_safety": 5.25 + }, + "on_frontier": false } ] } diff --git a/apps/site/src/components/HarnessModelMatrix.tsx b/apps/site/src/components/HarnessModelMatrix.tsx index 1735045..661cbb6 100644 --- a/apps/site/src/components/HarnessModelMatrix.tsx +++ b/apps/site/src/components/HarnessModelMatrix.tsx @@ -10,7 +10,7 @@ import { frontierKeys, } from "@/lib/board"; import { heat, heatText, norm } from "@/lib/color"; -import { formatUsd, formatScore } from "@/lib/format"; +import { formatUsd, formatCost, formatScore } from "@/lib/format"; const METRICS: { id: Metric; label: string; caption: string }[] = [ { @@ -175,7 +175,7 @@ function MatrixCell({ const fg = v === null ? "#6e6e7a" : heatText(t); const title = `${cell.model_id} × ${cell.stack_id}\n` + - `score ${cell.mean_score ?? "—"} · ${formatUsd( + `score ${cell.mean_score ?? "—"} · ${formatCost( cell.mean_cost_usd )}/task · ` + `${ diff --git a/apps/site/src/components/PerTaskWinners.tsx b/apps/site/src/components/PerTaskWinners.tsx index 666e04d..ff191c0 100644 --- a/apps/site/src/components/PerTaskWinners.tsx +++ b/apps/site/src/components/PerTaskWinners.tsx @@ -1,5 +1,5 @@ import type { Board, Cell } from "@/lib/board"; -import { formatUsd, formatScore } from "@/lib/format"; +import { formatCost, formatScore } from "@/lib/format"; /** * Per-task winners — because the best stack is NOT the same for every task. @@ -61,7 +61,7 @@ export function PerTaskWinners({ board }: { board: Board }) { {formatScore(r.score)} - {formatUsd(r.cost)} + {formatCost(r.cost)} ))} diff --git a/apps/site/src/components/StackDrawer.tsx b/apps/site/src/components/StackDrawer.tsx index 9868359..d92b01e 100644 --- a/apps/site/src/components/StackDrawer.tsx +++ b/apps/site/src/components/StackDrawer.tsx @@ -2,7 +2,7 @@ import { useEffect } from "react"; import type { Board, Cell } from "@/lib/board"; -import { formatUsd, formatScore } from "@/lib/format"; +import { formatCost, formatScore } from "@/lib/format"; /** * Right-side drawer: everything about one stack (model × harness) in one place — @@ -60,7 +60,7 @@ export function StackDrawer({ value: cell.quality_per_dollar === null ? "—" : grp(cell.quality_per_dollar), }, - { label: "Cost / task", value: formatUsd(cell.mean_cost_usd) }, + { label: "Cost / task", value: formatCost(cell.mean_cost_usd) }, { label: "Speed", value: cell.mean_latency_ms diff --git a/apps/site/src/components/StackMasterTable.tsx b/apps/site/src/components/StackMasterTable.tsx index 267286c..352b10c 100644 --- a/apps/site/src/components/StackMasterTable.tsx +++ b/apps/site/src/components/StackMasterTable.tsx @@ -2,7 +2,7 @@ import { useMemo, useState } from "react"; import type { Board, Cell } from "@/lib/board"; -import { formatUsd, formatScore } from "@/lib/format"; +import { formatCost, formatScore } from "@/lib/format"; // Locale-independent thousands separator — `toLocaleString()` differs between // the Node server render and the browser, which breaks hydration. @@ -174,7 +174,7 @@ export function StackMasterTable({ board }: { board: Board }) {