From cca66ca230e7c64c0997346732e85d6f734dccdc Mon Sep 17 00:00:00 2001 From: gogocat Date: Wed, 3 Jun 2026 18:36:09 +0300 Subject: [PATCH 1/2] =?UTF-8?q?feat(harness):=20opencode=20CLI=20harness?= =?UTF-8?q?=20=E2=80=94=20image=20+=20recipe=20+=20config=5Ffiles=20launch?= =?UTF-8?q?er?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third agentic harness (model-agnostic, sst/opencode). Uses opencode's BUILT-IN openai provider with a baseURL override → our proxy (the provider is bundled in the binary, so the no-egress sandbox needs no run-time npm). Built + isolation- smoked end-to-end (opencode run → proxy → wrote a file in the sandbox). Adds the config_files launcher write (DockerHarnessLauncher._write_config_files): a harness's config (opencode.json now; codex config.toml next) is written into the workspace BEFORE the git base commit, so the harness finds it AND it stays out of the captured patch. Shared infra for opencode / codex / Crush. - infra/docker/harness-opencode: node base + npm opencode-ai@1.15.13 (uid-1000 `node` user matches the host bind owner). - _opencode_invocation recipe (opencode.json via config_files; -m openai/). Promoted opencode proven → out of _PENDING_RECIPES. - stacks/opencode/stack.yaml: command `opencode run`, L1+L2+L3+L4. - tests: opencode recipe-proven + config_files-written-then-excluded-from-patch; supported_harnesses = {aider, goose, opencode}. 729 tests green; ruff + mypy --strict clean; 12/12 stack specs valid. Refs: rfc-006-stack-executor Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile | 7 +- .../src/orchestrator/stack_executor.py | 58 ++++++++++++++++- .../eval-core-py/tests/test_stack_executor.py | 45 ++++++++++++- infra/docker/harness-opencode/Dockerfile | 64 +++++++++++++++++++ stacks/opencode/stack.yaml | 14 ++-- 5 files changed, 174 insertions(+), 14 deletions(-) create mode 100644 infra/docker/harness-opencode/Dockerfile diff --git a/Makefile b/Makefile index 939030e..8580871 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ endif smoke-run smoke-dry resume postmortem \ eval-core-test litellm-up litellm-down stack-up stack-down stack-status \ openrouter-smoke env-check \ - harness-image-aider harness-image-goose sandbox-net-up + harness-image-aider harness-image-goose harness-image-opencode sandbox-net-up demo-run: python -m pollmevals_eval_core.demo_run --tasks evals/tasks --output artifacts @@ -123,6 +123,11 @@ harness-image-aider: harness-image-goose: docker build -t pollmevals-harness-goose:0.1.0 infra/docker/harness-goose/ +# Build the opencode harness image (RFC-006 Phase 5). No spend — node base + +# npm opencode-ai. Model-agnostic via opencode's built-in openai provider (proxy). +harness-image-opencode: + docker build -t pollmevals-harness-opencode:0.1.0 infra/docker/harness-opencode/ + # Build the Python eval sandbox image (Phase 5 — PythonCorrectnessEvaluator, # runs BigCodeBench unittest suites). No spend — pulls base + pip scientific stack. eval-image-py: diff --git a/apps/eval-core-py/src/orchestrator/stack_executor.py b/apps/eval-core-py/src/orchestrator/stack_executor.py index c52c6f8..325d15a 100644 --- a/apps/eval-core-py/src/orchestrator/stack_executor.py +++ b/apps/eval-core-py/src/orchestrator/stack_executor.py @@ -36,6 +36,7 @@ import asyncio import contextlib +import json import logging import os import re @@ -264,12 +265,48 @@ def _goose_invocation( ) +def _opencode_invocation( + proxy_base_url: str, api_key: str, model_alias: str, prompt: str +) -> ProxyInvocation: + """opencode (sst) recipe — PROVEN (2026-06-03 isolation smoke; memory). + + opencode is model-agnostic via AI-SDK providers. Its custom-provider packages + (e.g. @ai-sdk/openai-compatible) resolve over npm at RUN time, which the + no-egress sandbox can't do — so we use the BUILT-IN ``openai`` provider + (bundled in the binary) with a ``baseURL`` override pointed at our proxy. The + provider config rides an ``opencode.json`` written into the workspace via + ``config_files`` (the launcher writes it before the git base commit, so it + stays out of the captured patch). The model is selected as ``openai/``; + the prompt is the positional arg to ``opencode run`` (stack.yaml). + """ + base = proxy_base_url.rstrip("/") + config = json.dumps( + { + "$schema": "https://opencode.ai/config.json", + "provider": { + "openai": { + "options": {"baseURL": f"{base}/v1"}, + "models": {model_alias: {"name": model_alias}}, + } + }, + }, + indent=2, + ) + return ProxyInvocation( + env={"OPENAI_API_KEY": api_key}, + config_files={"opencode.json": config}, + extra_args=["-m", f"openai/{model_alias}"], + prompt_args=[prompt], + ) + + # Proven recipes (validated end-to-end via the proxy). aider is the RFC-006 -# first slice (aider x qwen x be_01); goose is the second harness (2026-06-03), -# a model-agnostic peer that runs the same coder models for a clean comparison. +# first slice (aider x qwen x be_01); goose (2026-06-03) and opencode (2026-06-03) +# are model-agnostic peers that run the same coder models for a clean comparison. _PROVEN_RECIPES: dict[str, _RecipeBuilder] = { "aider": _aider_invocation, "goose": _goose_invocation, + "opencode": _opencode_invocation, } # Known harnesses whose recipe is proven in spikes but lands at its per-stack @@ -278,7 +315,6 @@ def _goose_invocation( { "claude-code", "codex", - "opencode", "openhands", "hermes", "cline", @@ -515,8 +551,24 @@ def _capture_patch(self, workspace: Path, base_sha: str) -> str: self._git(workspace, "add", "-A") return self._git(workspace, "diff", "--cached", base_sha).stdout + @staticmethod + def _write_config_files(workspace: Path, config_files: dict[str, str]) -> None: + """Write a harness's config files (e.g. opencode.json, codex config.toml) + into the workspace BEFORE the base commit. + + Two effects: (a) the harness finds its config when it runs, and (b) the + files land in the git base, so they're EXCLUDED from the captured patch + (the diff is the candidate's real edits, not its config scaffolding). + Relative paths only; parent dirs are created. + """ + for relpath, content in config_files.items(): + dest = workspace / relpath + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(content, encoding="utf-8") + def _run_sync(self, plan: HarnessRunPlan) -> HarnessRunOutcome: workspace = plan.mount_dir + self._write_config_files(workspace, plan.config_files) base_sha = self._ensure_git_base(workspace) client = self._ensure_client() diff --git a/apps/eval-core-py/tests/test_stack_executor.py b/apps/eval-core-py/tests/test_stack_executor.py index c7bf722..86e7210 100644 --- a/apps/eval-core-py/tests/test_stack_executor.py +++ b/apps/eval-core-py/tests/test_stack_executor.py @@ -7,6 +7,7 @@ from __future__ import annotations import dataclasses +import json from datetime import UTC, datetime from decimal import Decimal from pathlib import Path @@ -217,7 +218,7 @@ def test_goose_strips_trailing_slash(self) -> None: ) assert inv.env["OPENAI_HOST"] == "http://h:4000" - @pytest.mark.parametrize("cli", ["claude-code", "codex", "openhands", "opencode"]) + @pytest.mark.parametrize("cli", ["claude-code", "codex", "openhands"]) def test_known_but_pending_harness_raises_pending(self, cli: str) -> None: with pytest.raises(HarnessRecipePending, match="Phase 5"): build_proxy_invocation( @@ -236,8 +237,28 @@ def test_none_cli_raises_unsupported(self) -> None: None, proxy_base_url="x", api_key="k", model_alias="m", prompt="p" ) - def test_supported_harnesses_are_aider_and_goose(self) -> None: - assert supported_harnesses() == frozenset({"aider", "goose"}) + def test_opencode_recipe_is_proven(self) -> None: + inv = build_proxy_invocation( + "opencode", + proxy_base_url="http://pollmevals-litellm-proxy:4000", + api_key="sk-local-xyz", + model_alias="qwen-3-14b", + prompt="do the thing", + ) + assert inv.env["OPENAI_API_KEY"] == "sk-local-xyz" + assert inv.extra_args == ["-m", "openai/qwen-3-14b"] + assert inv.prompt_args == ["do the thing"] + # config rides opencode.json (built-in openai provider → our proxy). + assert "opencode.json" in inv.config_files + cfg = json.loads(inv.config_files["opencode.json"]) + assert ( + cfg["provider"]["openai"]["options"]["baseURL"] + == "http://pollmevals-litellm-proxy:4000/v1" + ) + assert "qwen-3-14b" in cfg["provider"]["openai"]["models"] + + def test_supported_harnesses_are_aider_goose_opencode(self) -> None: + assert supported_harnesses() == frozenset({"aider", "goose", "opencode"}) # --------------------------------------------------------------------------- @@ -441,6 +462,24 @@ def test_capture_empty_when_no_changes(self, tmp_path: Path) -> None: base = launcher._ensure_git_base(tmp_path) assert launcher._capture_patch(tmp_path, base).strip() == "" + def test_config_files_written_then_excluded_from_patch(self, tmp_path: Path) -> None: + launcher = DockerHarnessLauncher() + # harness config (incl. a nested path) is written into the workspace... + launcher._write_config_files( + tmp_path, + {"opencode.json": '{"provider":{}}\n', ".codex/config.toml": "model='x'\n"}, + ) + assert (tmp_path / "opencode.json").read_text() == '{"provider":{}}\n' + assert (tmp_path / ".codex" / "config.toml").exists() # nested dir created + # ...BEFORE the base commit, so the config lands in the base. + base = launcher._ensure_git_base(tmp_path) + # the harness then edits a real source file + (tmp_path / "solution.ts").write_text("export const x = 1;\n") + patch = launcher._capture_patch(tmp_path, base) + assert "solution.ts" in patch # the candidate's edit IS captured + assert "opencode.json" not in patch # config scaffolding is NOT in the patch + assert "config.toml" not in patch + def test_parse_aider_tokens(self) -> None: launcher = DockerHarnessLauncher() assert launcher._parse_tokens("Tokens: 1.2k sent, 850 received.") == (1200, 850) diff --git a/infra/docker/harness-opencode/Dockerfile b/infra/docker/harness-opencode/Dockerfile new file mode 100644 index 0000000..cd63bbf --- /dev/null +++ b/infra/docker/harness-opencode/Dockerfile @@ -0,0 +1,64 @@ +# syntax=docker/dockerfile:1.7 +# +# pollmevals-harness-opencode -- sandboxed opencode (sst) CLI image (RFC-006 +# Half A). Model-AGNOSTIC agentic CLI (built on the Vercel AI SDK with custom +# OpenAI-compatible providers), so it runs the SAME open coder models as +# aider/goose -- another clean "swap the harness, hold the model" column. +# +# Candidate-side shape (same as aider/goose, OPPOSITE of the Half B evaluator): +# * /workspace WRITABLE (the harness edits files to produce a patch) +# * joins the `pollmevals-sandbox` INTERNAL net -> reaches ONLY the proxy. +# +# Pinned dependencies (Library-first; bump deliberately): +# node 22-slim (opencode ships a Bun-compiled binary via npm +# optionalDependencies, opencode-linux-) +# opencode-ai 1.15.13 (npm; pulls the platform binary) +# git (Debian stable) -- opencode tracks edits; we capture the diff via git +# +# opencode headless gotchas: +# * opencode resolves custom AI-SDK provider packages (e.g. +# @ai-sdk/openai-compatible) at RUN time via npm -- which the no-egress +# sandbox cannot do. So we pre-install that provider at BUILD time and the +# recipe points opencode at it (or uses a built-in provider). The opencode.json +# config is written by the recipe (config_files), not baked (run-time proxy URL). +# * `opencode run -m / --dangerously-skip-permissions ""` +# is the one-shot headless form; `--print-logs`/`--format json` for output. +# +# Build: docker build -t pollmevals-harness-opencode:0.1.0 infra/docker/harness-opencode/ +# (or: make harness-image-opencode) +# Smoke: docker run --rm pollmevals-harness-opencode:0.1.0 # -> prints opencode version + +FROM node:22-slim AS base + +# git: opencode tracks edits; the launcher captures the diff host-side via git. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Install opencode globally (pulls the platform binary via optionalDependencies). +# The recipe uses opencode's BUILT-IN `openai` provider with a baseURL override +# pointed at our proxy — that provider is bundled in the binary, so the no-egress +# run-time container needs no npm at all (verified: a smoke run edited a file with +# zero network beyond the proxy). Pinned. +RUN npm install -g --no-fund --no-audit opencode-ai@1.15.13 \ + && npm cache clean --force \ + && opencode --version + +# Non-root user: defense-in-depth on top of --cap-drop=ALL and +# --security-opt=no-new-privileges. The node base already ships a uid-1000 `node` +# user, which matches the host bind owner so the produced patch is writable back. +USER node +WORKDIR /workspace + +# Default git identity so a fresh snapshot is a valid repo to diff against. +RUN git config --global user.email "harness@pollmevals.local" \ + && git config --global user.name "pollmevals-harness" \ + && git config --global init.defaultBranch main \ + && git config --global --add safe.directory /workspace + +# opencode writes telemetry/state under $HOME; keep it inside the container. +ENV OPENCODE_DISABLE_AUTOUPDATE=1 + +# Default command is a harmless version probe; DockerHarnessLauncher overrides +# `command` with the full `opencode run ...` invocation. +CMD ["opencode", "--version"] diff --git a/stacks/opencode/stack.yaml b/stacks/opencode/stack.yaml index bf0e404..4931fb0 100644 --- a/stacks/opencode/stack.yaml +++ b/stacks/opencode/stack.yaml @@ -4,13 +4,13 @@ name: OpenCode base_model_slug: configurable agent_cli: opencode -# Proven headless-via-proxy 2026-06-02 (memory: research-cli-harness-execution). -# Model-agnostic. Config opencode.json: -# provider.litellm = { npm: "@ai-sdk/openai-compatible", -# options: { baseURL: "http://localhost:4000/v1", -# apiKey: "{env:LITELLM_MASTER_KEY}" }, -# models: { "": { name: "" } } } -# Run: opencode run "" -m litellm/ +# Model-agnostic (Vercel AI SDK). Recipe codified + smoked 2026-06-03 +# (_opencode_invocation in stack_executor.py; memory: research-cli-harness-execution). +# Uses opencode's BUILT-IN `openai` provider (bundled in the binary → no run-time +# npm fetch, works on the no-egress sandbox) with a baseURL override to our proxy. +# config_files writes opencode.json: provider.openai.options.baseURL=/v1 + +# models.{}; env OPENAI_API_KEY=$LITELLM_MASTER_KEY. +# Run: opencode run -m openai/ "" layers: L0_bare_llm: false From b1a4ddb81a37928bf7e75a702d18b5d693e5714c Mon Sep 17 00:00:00 2001 From: gogocat Date: Wed, 3 Jun 2026 18:36:31 +0300 Subject: [PATCH 2/2] =?UTF-8?q?feat(board):=20opencode=20harness=20column?= =?UTF-8?q?=20=E2=80=94=20honest=201/4=20(tool-format=20compat=20finding)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runs the opencode grid (--add-stack opencode, same 4 coder models, 2 seeds) and merges the column into board.json (now 23/29 scored). opencode × qwen-3-14b = 6.92; the other 3 coders FAILED — a real harness×model-compat finding: qwen3-coder-30b / codestral / devstral emit tool calls in an XML-ish format (`...`) that opencode's openai-provider doesn't parse → no file edit → NO_PATCH. Not a timeout (the run finishes in ~5s). opencode's tool-calling is more format-sensitive than goose/aider, which tolerate these models. Shown honestly (FAILED cells render "—"), never hidden. Refs: rfc-006-stack-executor Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/eval-core-py/scripts/build_real_board.py | 8 +- apps/site/public/board.json | 91 +++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/apps/eval-core-py/scripts/build_real_board.py b/apps/eval-core-py/scripts/build_real_board.py index 6f57440..baee452 100644 --- a/apps/eval-core-py/scripts/build_real_board.py +++ b/apps/eval-core-py/scripts/build_real_board.py @@ -87,8 +87,14 @@ # comparable pair — isolating the harness variable (aider L4 vs goose L2 on # identical models). Diverge this list later if goose handles models aider can't. _GOOSE_MODELS = ["qwen-3-14b", "qwen3-coder-30b", "codestral", "devstral"] +# opencode (sst): model-agnostic, same coder models → directly comparable. +_OPENCODE_MODELS = ["qwen-3-14b", "qwen3-coder-30b", "codestral", "devstral"] # Per-stack candidate model lists for --add-stack (merge ONE harness column in). -_STACK_MODELS = {"aider": _AIDER_MODELS, "goose": _GOOSE_MODELS} +_STACK_MODELS = { + "aider": _AIDER_MODELS, + "goose": _GOOSE_MODELS, + "opencode": _OPENCODE_MODELS, +} _SEEDS = [1, 2] _TASK = "be_01_jwt_auth" _JUDGES = ["claude-sonnet-4-6-judge", "gpt-5-mini-judge", "gemini-3-flash"] diff --git a/apps/site/public/board.json b/apps/site/public/board.json index 1bfda81..5080d08 100644 --- a/apps/site/public/board.json +++ b/apps/site/public/board.json @@ -36,6 +36,18 @@ "L2_tools" ], "family": "agnostic" + }, + { + "stack_id": "opencode", + "name": "OpenCode", + "level": 4, + "layers": [ + "L1_system_prompt", + "L2_tools", + "L3_skills", + "L4_file_memory" + ], + "family": "agnostic" } ], "models": [ @@ -732,6 +744,85 @@ "type_safety": 5.25 }, "on_frontier": false + }, + { + "model_id": "codestral", + "stack_id": "opencode", + "mean_score": null, + "mean_cost_usd": 0.0, + "mean_latency_ms": 18721, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": null, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": {}, + "on_frontier": false + }, + { + "model_id": "devstral", + "stack_id": "opencode", + "mean_score": null, + "mean_cost_usd": 0.0, + "mean_latency_ms": 601029, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": null, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": {}, + "on_frontier": false + }, + { + "model_id": "qwen-3-14b", + "stack_id": "opencode", + "mean_score": 6.92, + "mean_cost_usd": 0.0, + "mean_latency_ms": 35845, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 6.92, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 7.5, + "correctness": 6.5, + "error_handling": 7.0, + "security_posture": 8.0, + "test_alignment": 6.5, + "type_safety": 6.0 + }, + "on_frontier": false + }, + { + "model_id": "qwen3-coder-30b", + "stack_id": "opencode", + "mean_score": null, + "mean_cost_usd": 0.0, + "mean_latency_ms": 20283, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": null, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": {}, + "on_frontier": false } ] }