diff --git a/Makefile b/Makefile index 1678d48..939030e 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ endif smoke-run smoke-dry resume postmortem \ eval-core-test litellm-up litellm-down stack-up stack-down stack-status \ openrouter-smoke env-check \ - harness-image-aider sandbox-net-up + harness-image-aider harness-image-goose sandbox-net-up demo-run: python -m pollmevals_eval_core.demo_run --tasks evals/tasks --output artifacts @@ -118,6 +118,11 @@ stack-status: harness-image-aider: docker build -t pollmevals-harness-aider:0.1.0 infra/docker/harness-aider/ +# Build the goose harness image (RFC-006 Phase 5). No spend — pulls base + +# the Block goose release binary. goose is model-agnostic (OpenAI-compatible). +harness-image-goose: + docker build -t pollmevals-harness-goose:0.1.0 infra/docker/harness-goose/ + # Build the Python eval sandbox image (Phase 5 — PythonCorrectnessEvaluator, # runs BigCodeBench unittest suites). No spend — pulls base + pip scientific stack. eval-image-py: diff --git a/apps/eval-core-py/scripts/build_real_board.py b/apps/eval-core-py/scripts/build_real_board.py index dc1ec85..6f57440 100644 --- a/apps/eval-core-py/scripts/build_real_board.py +++ b/apps/eval-core-py/scripts/build_real_board.py @@ -83,6 +83,12 @@ "grok-4", ] _AIDER_MODELS = ["qwen-3-14b", "qwen3-coder-30b", "codestral", "devstral"] +# goose runs the SAME coder models as aider, so each (model) gives a directly +# comparable pair — isolating the harness variable (aider L4 vs goose L2 on +# identical models). Diverge this list later if goose handles models aider can't. +_GOOSE_MODELS = ["qwen-3-14b", "qwen3-coder-30b", "codestral", "devstral"] +# Per-stack candidate model lists for --add-stack (merge ONE harness column in). +_STACK_MODELS = {"aider": _AIDER_MODELS, "goose": _GOOSE_MODELS} _SEEDS = [1, 2] _TASK = "be_01_jwt_auth" _JUDGES = ["claude-sonnet-4-6-judge", "gpt-5-mini-judge", "gemini-3-flash"] @@ -171,6 +177,14 @@ async def _main() -> int: "existing board.json (fills previously-failed cells without re-running " "the rest). e.g. --fill grok-4", ) + ap.add_argument( + "--add-stack", + default="", + help="run ONLY this stack's grid (on _STACK_MODELS[stack]) and MERGE its " + "new harness column (cells + harness metadata) into the existing " + "board.json, without re-spending on the other harnesses. e.g. " + "--add-stack goose", + ) args = ap.parse_args() os.chdir(REPO) @@ -275,6 +289,53 @@ def caller_for(stack_id: str) -> object: print(f" {c.stack_id} x {c.model_id}: score={c.mean_score} cost=${c.mean_cost_usd}") return 0 + # --add-stack: run ONLY this harness's grid and merge its NEW column (cells + + # harness metadata) into the existing board.json, without re-spending on the + # other harnesses. Unlike --fill (replace cells in place), this also unions + # the new harness into board.harnesses so the matrix renders the column. + if args.add_stack: + stack_id = args.add_stack + models = _STACK_MODELS.get(stack_id, _AIDER_MODELS) + print(f"ADD-STACK: {stack_id} x {models} → merge column into {out.name} ...") + result = await runner.run( + GridSpec( + run_hash=_RUN_HASH, + models=models, + tasks=[_TASK], + stacks=[stack_id], + seeds=_SEEDS, + task_timeout_s=task_timeout, + ) + ) + rows = _rows_from_result(result, _PRICING) + partial = build_board(rows, stacks_root=stacks_root, run_hash=_RUN_HASH, run_type="smoke") + existing = Board.model_validate_json(out.read_text(encoding="utf-8")) + # Replace-or-append cells by (model, stack); union harnesses + models. + by_key = {(c.model_id, c.stack_id): c for c in existing.cells} + for c in partial.cells: + by_key[(c.model_id, c.stack_id)] = c + h_by_id = {h.stack_id: h for h in existing.harnesses} + for h in partial.harnesses: + h_by_id[h.stack_id] = h + m_by_id = {m.model_id: m for m in existing.models} + for m in partial.models: + m_by_id.setdefault(m.model_id, m) + cells = list(by_key.values()) + scored_now = sum(1 for c in cells if c.mean_score is not None) + merged_board = existing.model_copy( + update={ + "cells": cells, + "harnesses": list(h_by_id.values()), + "models": list(m_by_id.values()), + "scored": scored_now > 0, + } + ) + out.write_text(merged_board.model_dump_json(indent=2) + "\n", encoding="utf-8") + print(f" board now {scored_now}/{len(cells)} cells scored. new {stack_id} cells:") + for c in partial.cells: + print(f" {c.stack_id} x {c.model_id}: score={c.mean_score} cost=${c.mean_cost_usd}") + return 0 + # Two specs so the grid is non-cartesian: raw-llm on every candidate, aider # only on the models that follow its edit format. Same run_hash + runner so # rows merge into one board. diff --git a/apps/eval-core-py/src/orchestrator/stack_executor.py b/apps/eval-core-py/src/orchestrator/stack_executor.py index f637498..c52c6f8 100644 --- a/apps/eval-core-py/src/orchestrator/stack_executor.py +++ b/apps/eval-core-py/src/orchestrator/stack_executor.py @@ -231,10 +231,45 @@ def _aider_invocation( ) +def _goose_invocation( + proxy_base_url: str, api_key: str, model_alias: str, prompt: str +) -> ProxyInvocation: + """Goose recipe — PROVEN (2026-06-03 isolation smoke; memory). + + Block's goose is model-agnostic via an OpenAI-compatible provider. Unlike + aider's single ``OPENAI_API_BASE``, goose splits the endpoint into + ``OPENAI_HOST`` (scheme+host, NO path) + ``OPENAI_BASE_PATH`` (the chat + route), and selects provider/model via ``GOOSE_PROVIDER`` / ``GOOSE_MODEL``. + ``GOOSE_DISABLE_KEYRING`` (no system keyring in a sandbox container) and + ``GOOSE_MODE=auto`` (never block waiting on a tool-call confirmation) are + baked into the image; set here too so the recipe is self-contained. The + model is chosen by env, so ``extra_args`` is empty; the prompt rides ``-t`` + and the ``goose run --no-session --with-builtin developer`` scaffolding + lives in stack.yaml ``execution.args``. + """ + base = proxy_base_url.rstrip("/") + return ProxyInvocation( + env={ + "GOOSE_PROVIDER": "openai", + "GOOSE_MODEL": model_alias, + "GOOSE_MODE": "auto", + "GOOSE_DISABLE_KEYRING": "1", + "OPENAI_API_KEY": api_key, + "OPENAI_HOST": base, + "OPENAI_BASE_PATH": "v1/chat/completions", + }, + config_files={}, + extra_args=[], + prompt_args=["-t", prompt], + ) + + # Proven recipes (validated end-to-end via the proxy). aider is the RFC-006 -# first slice (aider x qwen x be_01). +# first slice (aider x qwen x be_01); goose is the second harness (2026-06-03), +# a model-agnostic peer that runs the same coder models for a clean comparison. _PROVEN_RECIPES: dict[str, _RecipeBuilder] = { "aider": _aider_invocation, + "goose": _goose_invocation, } # Known harnesses whose recipe is proven in spikes but lands at its per-stack @@ -244,7 +279,6 @@ def _aider_invocation( "claude-code", "codex", "opencode", - "goose", "openhands", "hermes", "cline", diff --git a/apps/eval-core-py/tests/test_stack_executor.py b/apps/eval-core-py/tests/test_stack_executor.py index 6a6dab8..c7bf722 100644 --- a/apps/eval-core-py/tests/test_stack_executor.py +++ b/apps/eval-core-py/tests/test_stack_executor.py @@ -188,7 +188,36 @@ def test_aider_strips_trailing_slash(self) -> None: ) assert inv.env["OPENAI_API_BASE"] == "http://localhost:4000/v1" - @pytest.mark.parametrize("cli", ["claude-code", "codex", "goose", "openhands", "opencode"]) + def test_goose_recipe_is_proven(self) -> None: + inv = build_proxy_invocation( + "goose", + proxy_base_url="http://pollmevals-litellm-proxy:4000", + api_key="sk-local-xyz", + model_alias="qwen-3-14b", + prompt="do the thing", + ) + assert inv.env["GOOSE_PROVIDER"] == "openai" + assert inv.env["GOOSE_MODEL"] == "qwen-3-14b" + assert inv.env["OPENAI_API_KEY"] == "sk-local-xyz" + # goose splits the endpoint: host (NO path) + the chat route separately. + assert inv.env["OPENAI_HOST"] == "http://pollmevals-litellm-proxy:4000" + assert inv.env["OPENAI_BASE_PATH"] == "v1/chat/completions" + assert inv.env["GOOSE_DISABLE_KEYRING"] == "1" + assert inv.extra_args == [] # model selected by env, not a CLI flag + assert inv.prompt_args == ["-t", "do the thing"] + assert inv.config_files == {} + + def test_goose_strips_trailing_slash(self) -> None: + inv = build_proxy_invocation( + "goose", + proxy_base_url="http://h:4000/", + api_key="k", + model_alias="m", + prompt="p", + ) + assert inv.env["OPENAI_HOST"] == "http://h:4000" + + @pytest.mark.parametrize("cli", ["claude-code", "codex", "openhands", "opencode"]) def test_known_but_pending_harness_raises_pending(self, cli: str) -> None: with pytest.raises(HarnessRecipePending, match="Phase 5"): build_proxy_invocation( @@ -207,8 +236,8 @@ def test_none_cli_raises_unsupported(self) -> None: None, proxy_base_url="x", api_key="k", model_alias="m", prompt="p" ) - def test_supported_harnesses_is_aider_only_in_phase_1(self) -> None: - assert supported_harnesses() == frozenset({"aider"}) + def test_supported_harnesses_are_aider_and_goose(self) -> None: + assert supported_harnesses() == frozenset({"aider", "goose"}) # --------------------------------------------------------------------------- diff --git a/apps/site/public/board.json b/apps/site/public/board.json index 5dbd42c..1bfda81 100644 --- a/apps/site/public/board.json +++ b/apps/site/public/board.json @@ -26,6 +26,16 @@ "L0_bare_llm" ], "family": "baseline" + }, + { + "stack_id": "goose", + "name": "Goose", + "level": 2, + "layers": [ + "L1_system_prompt", + "L2_tools" + ], + "family": "agnostic" } ], "models": [ @@ -629,6 +639,99 @@ "type_safety": 5.25 }, "on_frontier": false + }, + { + "model_id": "codestral", + "stack_id": "goose", + "mean_score": null, + "mean_cost_usd": 0.0, + "mean_latency_ms": 9353, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": null, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": {}, + "on_frontier": false + }, + { + "model_id": "devstral", + "stack_id": "goose", + "mean_score": 7.12, + "mean_cost_usd": 0.0, + "mean_latency_ms": 200855, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 7.12, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 7.5, + "correctness": 7.75, + "error_handling": 7.0, + "security_posture": 8.0, + "test_alignment": 7.0, + "type_safety": 5.5 + }, + "on_frontier": false + }, + { + "model_id": "qwen-3-14b", + "stack_id": "goose", + "mean_score": 6.0, + "mean_cost_usd": 0.0, + "mean_latency_ms": 59121, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 6.0, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 7.0, + "correctness": 4.5, + "error_handling": 6.0, + "security_posture": 6.5, + "test_alignment": 7.0, + "type_safety": 5.0 + }, + "on_frontier": false + }, + { + "model_id": "qwen3-coder-30b", + "stack_id": "goose", + "mean_score": 7.17, + "mean_cost_usd": 0.0, + "mean_latency_ms": 48202, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 7.17, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 7.5, + "correctness": 8.0, + "error_handling": 7.5, + "security_posture": 8.25, + "test_alignment": 6.5, + "type_safety": 5.25 + }, + "on_frontier": false } ] } diff --git a/apps/site/src/components/HarnessModelMatrix.tsx b/apps/site/src/components/HarnessModelMatrix.tsx index 1735045..661cbb6 100644 --- a/apps/site/src/components/HarnessModelMatrix.tsx +++ b/apps/site/src/components/HarnessModelMatrix.tsx @@ -10,7 +10,7 @@ import { frontierKeys, } from "@/lib/board"; import { heat, heatText, norm } from "@/lib/color"; -import { formatUsd, formatScore } from "@/lib/format"; +import { formatUsd, formatCost, formatScore } from "@/lib/format"; const METRICS: { id: Metric; label: string; caption: string }[] = [ { @@ -175,7 +175,7 @@ function MatrixCell({ const fg = v === null ? "#6e6e7a" : heatText(t); const title = `${cell.model_id} × ${cell.stack_id}\n` + - `score ${cell.mean_score ?? "—"} · ${formatUsd( + `score ${cell.mean_score ?? "—"} · ${formatCost( cell.mean_cost_usd )}/task · ` + `${ diff --git a/apps/site/src/components/PerTaskWinners.tsx b/apps/site/src/components/PerTaskWinners.tsx index 666e04d..ff191c0 100644 --- a/apps/site/src/components/PerTaskWinners.tsx +++ b/apps/site/src/components/PerTaskWinners.tsx @@ -1,5 +1,5 @@ import type { Board, Cell } from "@/lib/board"; -import { formatUsd, formatScore } from "@/lib/format"; +import { formatCost, formatScore } from "@/lib/format"; /** * Per-task winners — because the best stack is NOT the same for every task. @@ -61,7 +61,7 @@ export function PerTaskWinners({ board }: { board: Board }) { {formatScore(r.score)} - {formatUsd(r.cost)} + {formatCost(r.cost)} ))} diff --git a/apps/site/src/components/StackDrawer.tsx b/apps/site/src/components/StackDrawer.tsx index 9868359..d92b01e 100644 --- a/apps/site/src/components/StackDrawer.tsx +++ b/apps/site/src/components/StackDrawer.tsx @@ -2,7 +2,7 @@ import { useEffect } from "react"; import type { Board, Cell } from "@/lib/board"; -import { formatUsd, formatScore } from "@/lib/format"; +import { formatCost, formatScore } from "@/lib/format"; /** * Right-side drawer: everything about one stack (model × harness) in one place — @@ -60,7 +60,7 @@ export function StackDrawer({ value: cell.quality_per_dollar === null ? "—" : grp(cell.quality_per_dollar), }, - { label: "Cost / task", value: formatUsd(cell.mean_cost_usd) }, + { label: "Cost / task", value: formatCost(cell.mean_cost_usd) }, { label: "Speed", value: cell.mean_latency_ms diff --git a/apps/site/src/components/StackMasterTable.tsx b/apps/site/src/components/StackMasterTable.tsx index 267286c..352b10c 100644 --- a/apps/site/src/components/StackMasterTable.tsx +++ b/apps/site/src/components/StackMasterTable.tsx @@ -2,7 +2,7 @@ import { useMemo, useState } from "react"; import type { Board, Cell } from "@/lib/board"; -import { formatUsd, formatScore } from "@/lib/format"; +import { formatCost, formatScore } from "@/lib/format"; // Locale-independent thousands separator — `toLocaleString()` differs between // the Node server render and the browser, which breaks hydration. @@ -174,7 +174,7 @@ export function StackMasterTable({ board }: { board: Board }) { {r.score === null ? "—" : formatScore(r.score)} - {formatUsd(r.cost)} + {formatCost(r.cost)} {r.latency ? (r.latency / 1000).toFixed(1) + "s" : "—"} diff --git a/apps/site/src/components/StackParetoChart.tsx b/apps/site/src/components/StackParetoChart.tsx index aa3e625..7c99ba3 100644 --- a/apps/site/src/components/StackParetoChart.tsx +++ b/apps/site/src/components/StackParetoChart.tsx @@ -27,7 +27,11 @@ export function StackParetoChart({ board }: { board: Board }) { const plotW = W - pad.l - pad.r; const plotH = H - pad.t - pad.b; - const scored = board.cells.filter((c) => c.mean_score !== null); + // Need both axes: a quality score AND a real (metered) cost. cost 0 ⇒ + // unmetered → can't honestly place it on the cost axis, so leave it off. + const scored = board.cells.filter( + (c) => c.mean_score !== null && c.mean_cost_usd > 0 + ); if (scored.length === 0) return null; const frontier = frontierKeys(board); const map = cellMap(board); diff --git a/apps/site/src/lib/board.ts b/apps/site/src/lib/board.ts index 6af8015..34a38ca 100644 --- a/apps/site/src/lib/board.ts +++ b/apps/site/src/lib/board.ts @@ -104,7 +104,12 @@ export function buildMatrix(board: Board): MatrixView { export type Metric = "quality_per_dollar" | "mean_score" | "mean_cost_usd"; export function metricValue(cell: Cell, metric: Metric): number | null { - if (metric === "mean_cost_usd") return cell.mean_cost_usd; + // cost 0 ⇒ unmetered (a real metered eval is never exactly $0); treat it as + // "no value" — exactly like an unscored quality cell — so it renders "—" and + // drops out of the color scale / best / frontier instead of masquerading as + // the cheapest (free) stack. Real cost lands via proxy-side reconciliation. + if (metric === "mean_cost_usd") + return cell.mean_cost_usd > 0 ? cell.mean_cost_usd : null; return cell[metric]; } @@ -135,11 +140,14 @@ export function metricRange( /** * Pareto frontier over cells: maximize quality, minimize cost. Unscored cells - * (no mean_score) are excluded — there's no quality axis to be on the frontier of. - * Mutates nothing; returns the set of frontier keys. + * (no mean_score) are excluded — there's no quality axis to be on the frontier + * of — and so are unmetered cells (cost 0), which would otherwise dominate as + * "free". Mutates nothing; returns the set of frontier keys. */ export function frontierKeys(board: Board): Set { - const scored = board.cells.filter((c) => c.mean_score !== null); + const scored = board.cells.filter( + (c) => c.mean_score !== null && c.mean_cost_usd > 0 + ); const out = new Set(); for (const c of scored) { const dominated = scored.some( diff --git a/apps/site/src/lib/format.ts b/apps/site/src/lib/format.ts index ab097e3..0eda192 100644 --- a/apps/site/src/lib/format.ts +++ b/apps/site/src/lib/format.ts @@ -9,6 +9,14 @@ export function formatUsd(value: string | number): string { return `$${n.toFixed(2)}`; } +/** A cell's cost, or "—" when unmetered. A real metered eval is never exactly + * $0, so cost === 0 means the token usage wasn't captured (e.g. a black-box + * harness that doesn't self-report; proxy-side reconciliation is the fix) — + * show "unknown", never a misleading "$0 / free". */ +export function formatCost(value: number): string { + return value > 0 ? formatUsd(value) : "—"; +} + /** Latency ms → human: "3.3s" or "847ms". */ export function formatLatency(ms: number): string { if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`; diff --git a/infra/docker/harness-goose/Dockerfile b/infra/docker/harness-goose/Dockerfile new file mode 100644 index 0000000..b8d2648 --- /dev/null +++ b/infra/docker/harness-goose/Dockerfile @@ -0,0 +1,106 @@ +# syntax=docker/dockerfile:1.7 +# +# pollmevals-harness-goose -- sandboxed Block "goose" CLI image (RFC-006 Half A, +# candidate side). Second harness after aider; goose is a model-AGNOSTIC agent +# (OpenAI-compatible provider), so it runs the SAME coder models aider does -> +# the cleanest "swap the harness, hold the model" comparison column. +# +# Same candidate-side shape as harness-aider (the OPPOSITE of the Half B +# evaluator image): +# * /workspace is WRITABLE (the harness edits files to produce a patch) +# * the container joins the `pollmevals-sandbox` INTERNAL network so it can +# reach ONLY the LiteLLM proxy (bastion) -- no general egress. goose talks +# to the proxy (which meters every token); it never holds the upstream key. +# +# Pinned dependencies (Library-first; bump deliberately, never floating): +# python 3.12-slim (base only -- goose is a standalone Rust binary; +# reuses the cached aider/eval-py base layer) +# goose v1.36.0 (Block release binary, pinned + downloaded +# directly -- the official download_cli.sh falls +# back to a stale mirror that lacks the arm64 asset, +# so we fetch the exact tarball ourselves) +# git (Debian stable) -- goose tracks edits; we capture the diff via git +# +# goose headless gotchas baked in (empirical spike + goose docs): +# * GOOSE_DISABLE_KEYRING=1 -- in a container there is no system keyring; +# without this goose errors on secret storage. Forces file-based secrets. +# * GOOSE_MODE=auto -- auto-approve tool calls; a headless run must never +# block waiting for an interactive confirmation. +# The provider/model/proxy env is injected at run time by the recipe +# (_goose_invocation in stack_executor.py), not baked here. +# +# Build: +# docker build -t pollmevals-harness-goose:0.1.0 infra/docker/harness-goose/ +# (or: make harness-image-goose) +# +# Run (referenced from DockerHarnessLauncher -- not invoked by humans directly): +# docker run --rm \ +# --network=pollmevals-sandbox \ +# --cap-drop=ALL --security-opt=no-new-privileges:true \ +# --memory=2g --pids-limit=256 \ +# -v $(pwd)/snapshot:/workspace \ +# -e GOOSE_PROVIDER=openai -e GOOSE_MODEL= \ +# -e OPENAI_API_KEY=$LITELLM_MASTER_KEY \ +# -e OPENAI_HOST=http://pollmevals-litellm-proxy:4000 \ +# -e OPENAI_BASE_PATH=v1/chat/completions \ +# pollmevals-harness-goose:0.1.0 \ +# goose run --no-session -t "" +# +# Smoke (no proxy needed): +# docker run --rm pollmevals-harness-goose:0.1.0 # -> prints goose version + +FROM python:3.12-slim AS base + +# curl + bzip2 + ca-certificates: fetch + extract the goose release tarball. +# git: goose tracks edits; the launcher captures the diff host-side via git. +# libxcb1 + libdbus-1-3: the goose binary dynamically links the keyring stack +# even when GOOSE_DISABLE_KEYRING routes secrets to a file -- without these +# shared objects the binary fails to start at all. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + curl ca-certificates bzip2 git libxcb1 libdbus-1-3 \ + && rm -rf /var/lib/apt/lists/* + +# Install goose (root -> /usr/local/bin, on PATH for all users). Pinned tarball +# fetched directly: arch from BuildKit's TARGETARCH (arm64->aarch64, amd64-> +# x86_64), gnu (glibc) variant to match the Debian base. Pulled at BUILD time so +# the run-time container needs no network beyond the proxy. +ARG TARGETARCH +ARG GOOSE_VERSION=v1.36.0 +RUN set -eux; \ + case "${TARGETARCH}" in \ + arm64) GARCH=aarch64 ;; \ + amd64) GARCH=x86_64 ;; \ + *) echo "unsupported TARGETARCH: ${TARGETARCH}"; exit 1 ;; \ + esac; \ + url="https://github.com/block/goose/releases/download/${GOOSE_VERSION}/goose-${GARCH}-unknown-linux-gnu.tar.bz2"; \ + curl -fsSL -o /tmp/goose.tar.bz2 "$url"; \ + mkdir -p /tmp/goose-x; \ + tar -xjf /tmp/goose.tar.bz2 -C /tmp/goose-x; \ + find /tmp/goose-x -maxdepth 2 -type f -name goose -exec cp {} /usr/local/bin/goose \; ; \ + find /tmp/goose-x -maxdepth 2 -type f -name goosed -exec cp {} /usr/local/bin/goosed \; ; \ + chmod +x /usr/local/bin/goose; \ + rm -rf /tmp/goose.tar.bz2 /tmp/goose-x; \ + /usr/local/bin/goose --version + +# Non-root user: defense-in-depth on top of --cap-drop=ALL and +# --security-opt=no-new-privileges. uid 1000 matches the host bind owner so the +# produced patch is writable back on the host snapshot. +RUN useradd --create-home --uid 1000 harness +USER harness +WORKDIR /workspace + +# Default git identity so a fresh snapshot is a valid repo to diff against. +RUN git config --global user.email "harness@pollmevals.local" \ + && git config --global user.name "pollmevals-harness" \ + && git config --global init.defaultBranch main \ + && git config --global --add safe.directory /workspace + +# Headless defaults (see header). Provider/model/proxy are injected by the recipe. +ENV GOOSE_DISABLE_KEYRING=1 \ + GOOSE_MODE=auto + +# Default command is a harmless version probe; DockerHarnessLauncher overrides +# `command` with the full `goose run ...` invocation. No restrictive ENTRYPOINT +# so the launcher can pass either an argv list or an explicit sh -c wrapper. +CMD ["goose", "--version"] diff --git a/stacks/goose/stack.yaml b/stacks/goose/stack.yaml index 0174f85..99424f5 100644 --- a/stacks/goose/stack.yaml +++ b/stacks/goose/stack.yaml @@ -4,12 +4,16 @@ name: Goose base_model_slug: configurable agent_cli: goose -# Proven headless-via-proxy 2026-06-02 (memory: research-cli-harness-execution). -# Model-agnostic (AAIF / ex-Block), MCP-native tools. Env: -# GOOSE_PROVIDER=openai GOOSE_MODEL= -# OPENAI_API_KEY=$LITELLM_MASTER_KEY -# OPENAI_HOST=http://localhost:4000 OPENAI_BASE_PATH=v1/chat/completions -# Run: goose run --no-session -t "" +# Block's goose: a system-prompted agent (L1) driving the built-in "developer" +# tool extension — shell + file editor (L2). Model-agnostic (OpenAI-compatible +# provider), so it runs the same coder models as aider; the board orders +# harnesses by max true layer, placing goose (L2) between raw-llm (L0) and +# aider (L4). Recipe codified + smoked end-to-end 2026-06-03 +# (_goose_invocation in stack_executor.py; memory: research-cli-harness-execution): +# env GOOSE_PROVIDER=openai GOOSE_MODEL= GOOSE_MODE=auto +# GOOSE_DISABLE_KEYRING=1 OPENAI_API_KEY=$LITELLM_MASTER_KEY +# OPENAI_HOST=http://:4000 OPENAI_BASE_PATH=v1/chat/completions +# run goose run --no-session --with-builtin developer -t "" layers: L0_bare_llm: false @@ -26,7 +30,13 @@ execution: command: goose args: - run - - --no-session + - --no-session # automated run, no session-file persistence + - --with-builtin + - developer # enable the file-editor + shell tools (writes the patch) + - --max-turns + - "25" # bound the agent loop (single-file middleware scope) + - --max-tool-repetitions + - "3" # stop identical-tool loops (aider's llama/deepseek failure mode) input_contract: receives: - task_prompt