feat(harness): opencode CLI harness + config_files launcher (RFC-006 Phase 5)

Makefile

@@ -13,7 +13,7 @@ endif
         smoke-run smoke-dry resume postmortem \
         eval-core-test litellm-up litellm-down stack-up stack-down stack-status \
         openrouter-smoke env-check \
-        harness-image-aider harness-image-goose sandbox-net-up
+        harness-image-aider harness-image-goose harness-image-opencode sandbox-net-up
 
 demo-run:
 	python -m pollmevals_eval_core.demo_run --tasks evals/tasks --output artifacts
@@ -123,6 +123,11 @@ harness-image-aider:
 harness-image-goose:
 	docker build -t pollmevals-harness-goose:0.1.0 infra/docker/harness-goose/
 
+# Build the opencode harness image (RFC-006 Phase 5). No spend — node base +
+# npm opencode-ai. Model-agnostic via opencode's built-in openai provider (proxy).
+harness-image-opencode:
+	docker build -t pollmevals-harness-opencode:0.1.0 infra/docker/harness-opencode/
+
 # Build the Python eval sandbox image (Phase 5 — PythonCorrectnessEvaluator,
 # runs BigCodeBench unittest suites). No spend — pulls base + pip scientific stack.
 eval-image-py:

apps/eval-core-py/scripts/build_real_board.py

@@ -87,8 +87,14 @@
 # comparable pair — isolating the harness variable (aider L4 vs goose L2 on
 # identical models). Diverge this list later if goose handles models aider can't.
 _GOOSE_MODELS = ["qwen-3-14b", "qwen3-coder-30b", "codestral", "devstral"]
+# opencode (sst): model-agnostic, same coder models → directly comparable.
+_OPENCODE_MODELS = ["qwen-3-14b", "qwen3-coder-30b", "codestral", "devstral"]
 # Per-stack candidate model lists for --add-stack (merge ONE harness column in).
-_STACK_MODELS = {"aider": _AIDER_MODELS, "goose": _GOOSE_MODELS}
+_STACK_MODELS = {
+    "aider": _AIDER_MODELS,
+    "goose": _GOOSE_MODELS,
+    "opencode": _OPENCODE_MODELS,
+}
 _SEEDS = [1, 2]
 _TASK = "be_01_jwt_auth"
 _JUDGES = ["claude-sonnet-4-6-judge", "gpt-5-mini-judge", "gemini-3-flash"]

apps/eval-core-py/src/orchestrator/stack_executor.py

@@ -36,6 +36,7 @@
 
 import asyncio
 import contextlib
+import json
 import logging
 import os
 import re
@@ -264,12 +265,48 @@ def _goose_invocation(
     )
 
 
+def _opencode_invocation(
+    proxy_base_url: str, api_key: str, model_alias: str, prompt: str
+) -> ProxyInvocation:
+    """opencode (sst) recipe — PROVEN (2026-06-03 isolation smoke; memory).
+
+    opencode is model-agnostic via AI-SDK providers. Its custom-provider packages
+    (e.g. @ai-sdk/openai-compatible) resolve over npm at RUN time, which the
+    no-egress sandbox can't do — so we use the BUILT-IN ``openai`` provider
+    (bundled in the binary) with a ``baseURL`` override pointed at our proxy. The
+    provider config rides an ``opencode.json`` written into the workspace via
+    ``config_files`` (the launcher writes it before the git base commit, so it
+    stays out of the captured patch). The model is selected as ``openai/<alias>``;
+    the prompt is the positional arg to ``opencode run`` (stack.yaml).
+    """
+    base = proxy_base_url.rstrip("/")
+    config = json.dumps(
+        {
+            "$schema": "https://opencode.ai/config.json",
+            "provider": {
+                "openai": {
+                    "options": {"baseURL": f"{base}/v1"},
+                    "models": {model_alias: {"name": model_alias}},
+                }
+            },
+        },
+        indent=2,
+    )
+    return ProxyInvocation(
+        env={"OPENAI_API_KEY": api_key},
+        config_files={"opencode.json": config},
+        extra_args=["-m", f"openai/{model_alias}"],
+        prompt_args=[prompt],
+    )
+
+
 # Proven recipes (validated end-to-end via the proxy). aider is the RFC-006
-# first slice (aider x qwen x be_01); goose is the second harness (2026-06-03),
-# a model-agnostic peer that runs the same coder models for a clean comparison.
+# first slice (aider x qwen x be_01); goose (2026-06-03) and opencode (2026-06-03)
+# are model-agnostic peers that run the same coder models for a clean comparison.
 _PROVEN_RECIPES: dict[str, _RecipeBuilder] = {
     "aider": _aider_invocation,
     "goose": _goose_invocation,
+    "opencode": _opencode_invocation,
 }
 
 # Known harnesses whose recipe is proven in spikes but lands at its per-stack
@@ -278,7 +315,6 @@ def _goose_invocation(
     {
         "claude-code",
         "codex",
-        "opencode",
         "openhands",
         "hermes",
         "cline",
@@ -515,8 +551,24 @@ def _capture_patch(self, workspace: Path, base_sha: str) -> str:
         self._git(workspace, "add", "-A")
         return self._git(workspace, "diff", "--cached", base_sha).stdout
 
+    @staticmethod
+    def _write_config_files(workspace: Path, config_files: dict[str, str]) -> None:
+        """Write a harness's config files (e.g. opencode.json, codex config.toml)
+        into the workspace BEFORE the base commit.
+
+        Two effects: (a) the harness finds its config when it runs, and (b) the
+        files land in the git base, so they're EXCLUDED from the captured patch
+        (the diff is the candidate's real edits, not its config scaffolding).
+        Relative paths only; parent dirs are created.
+        """
+        for relpath, content in config_files.items():
+            dest = workspace / relpath
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            dest.write_text(content, encoding="utf-8")
+
     def _run_sync(self, plan: HarnessRunPlan) -> HarnessRunOutcome:
         workspace = plan.mount_dir
+        self._write_config_files(workspace, plan.config_files)
         base_sha = self._ensure_git_base(workspace)
 
         client = self._ensure_client()

apps/eval-core-py/tests/test_stack_executor.py

@@ -7,6 +7,7 @@
 from __future__ import annotations
 
 import dataclasses
+import json
 from datetime import UTC, datetime
 from decimal import Decimal
 from pathlib import Path
@@ -217,7 +218,7 @@ def test_goose_strips_trailing_slash(self) -> None:
         )
         assert inv.env["OPENAI_HOST"] == "http://h:4000"
 
-    @pytest.mark.parametrize("cli", ["claude-code", "codex", "openhands", "opencode"])
+    @pytest.mark.parametrize("cli", ["claude-code", "codex", "openhands"])
     def test_known_but_pending_harness_raises_pending(self, cli: str) -> None:
         with pytest.raises(HarnessRecipePending, match="Phase 5"):
             build_proxy_invocation(
@@ -236,8 +237,28 @@ def test_none_cli_raises_unsupported(self) -> None:
                 None, proxy_base_url="x", api_key="k", model_alias="m", prompt="p"
             )
 
-    def test_supported_harnesses_are_aider_and_goose(self) -> None:
-        assert supported_harnesses() == frozenset({"aider", "goose"})
+    def test_opencode_recipe_is_proven(self) -> None:
+        inv = build_proxy_invocation(
+            "opencode",
+            proxy_base_url="http://pollmevals-litellm-proxy:4000",
+            api_key="sk-local-xyz",
+            model_alias="qwen-3-14b",
+            prompt="do the thing",
+        )
+        assert inv.env["OPENAI_API_KEY"] == "sk-local-xyz"
+        assert inv.extra_args == ["-m", "openai/qwen-3-14b"]
+        assert inv.prompt_args == ["do the thing"]
+        # config rides opencode.json (built-in openai provider → our proxy).
+        assert "opencode.json" in inv.config_files
+        cfg = json.loads(inv.config_files["opencode.json"])
+        assert (
+            cfg["provider"]["openai"]["options"]["baseURL"]
+            == "http://pollmevals-litellm-proxy:4000/v1"
+        )
+        assert "qwen-3-14b" in cfg["provider"]["openai"]["models"]
+
+    def test_supported_harnesses_are_aider_goose_opencode(self) -> None:
+        assert supported_harnesses() == frozenset({"aider", "goose", "opencode"})
 
 
 # ---------------------------------------------------------------------------
@@ -441,6 +462,24 @@ def test_capture_empty_when_no_changes(self, tmp_path: Path) -> None:
         base = launcher._ensure_git_base(tmp_path)
         assert launcher._capture_patch(tmp_path, base).strip() == ""
 
+    def test_config_files_written_then_excluded_from_patch(self, tmp_path: Path) -> None:
+        launcher = DockerHarnessLauncher()
+        # harness config (incl. a nested path) is written into the workspace...
+        launcher._write_config_files(
+            tmp_path,
+            {"opencode.json": '{"provider":{}}\n', ".codex/config.toml": "model='x'\n"},
+        )
+        assert (tmp_path / "opencode.json").read_text() == '{"provider":{}}\n'
+        assert (tmp_path / ".codex" / "config.toml").exists()  # nested dir created
+        # ...BEFORE the base commit, so the config lands in the base.
+        base = launcher._ensure_git_base(tmp_path)
+        # the harness then edits a real source file
+        (tmp_path / "solution.ts").write_text("export const x = 1;\n")
+        patch = launcher._capture_patch(tmp_path, base)
+        assert "solution.ts" in patch  # the candidate's edit IS captured
+        assert "opencode.json" not in patch  # config scaffolding is NOT in the patch
+        assert "config.toml" not in patch
+
     def test_parse_aider_tokens(self) -> None:
         launcher = DockerHarnessLauncher()
         assert launcher._parse_tokens("Tokens: 1.2k sent, 850 received.") == (1200, 850)

apps/site/public/board.json

@@ -36,6 +36,18 @@
         "L2_tools"
       ],
       "family": "agnostic"
+    },
+    {
+      "stack_id": "opencode",
+      "name": "OpenCode",
+      "level": 4,
+      "layers": [
+        "L1_system_prompt",
+        "L2_tools",
+        "L3_skills",
+        "L4_file_memory"
+      ],
+      "family": "agnostic"
     }
   ],
   "models": [
@@ -732,6 +744,85 @@
         "type_safety": 5.25
       },
       "on_frontier": false
+    },
+    {
+      "model_id": "codestral",
+      "stack_id": "opencode",
+      "mean_score": null,
+      "mean_cost_usd": 0.0,
+      "mean_latency_ms": 18721,
+      "pass_hat_k": null,
+      "quality_per_dollar": null,
+      "per_task": {
+        "be_01_jwt_auth": {
+          "score": null,
+          "cost_usd": 0.0,
+          "pass_hat_k": null
+        }
+      },
+      "per_criterion": {},
+      "on_frontier": false
+    },
+    {
+      "model_id": "devstral",
+      "stack_id": "opencode",
+      "mean_score": null,
+      "mean_cost_usd": 0.0,
+      "mean_latency_ms": 601029,
+      "pass_hat_k": null,
+      "quality_per_dollar": null,
+      "per_task": {
+        "be_01_jwt_auth": {
+          "score": null,
+          "cost_usd": 0.0,
+          "pass_hat_k": null
+        }
+      },
+      "per_criterion": {},
+      "on_frontier": false
+    },
+    {
+      "model_id": "qwen-3-14b",
+      "stack_id": "opencode",
+      "mean_score": 6.92,
+      "mean_cost_usd": 0.0,
+      "mean_latency_ms": 35845,
+      "pass_hat_k": null,
+      "quality_per_dollar": null,
+      "per_task": {
+        "be_01_jwt_auth": {
+          "score": 6.92,
+          "cost_usd": 0.0,
+          "pass_hat_k": null
+        }
+      },
+      "per_criterion": {
+        "code_clarity": 7.5,
+        "correctness": 6.5,
+        "error_handling": 7.0,
+        "security_posture": 8.0,
+        "test_alignment": 6.5,
+        "type_safety": 6.0
+      },
+      "on_frontier": false
+    },
+    {
+      "model_id": "qwen3-coder-30b",
+      "stack_id": "opencode",
+      "mean_score": null,
+      "mean_cost_usd": 0.0,
+      "mean_latency_ms": 20283,
+      "pass_hat_k": null,
+      "quality_per_dollar": null,
+      "per_task": {
+        "be_01_jwt_auth": {
+          "score": null,
+          "cost_usd": 0.0,
+          "pass_hat_k": null
+        }
+      },
+      "per_criterion": {},
+      "on_frontier": false
     }
   ]
 }

infra/docker/harness-opencode/Dockerfile

@@ -0,0 +1,64 @@
+# syntax=docker/dockerfile:1.7
+#
+# pollmevals-harness-opencode -- sandboxed opencode (sst) CLI image (RFC-006
+# Half A). Model-AGNOSTIC agentic CLI (built on the Vercel AI SDK with custom
+# OpenAI-compatible providers), so it runs the SAME open coder models as
+# aider/goose -- another clean "swap the harness, hold the model" column.
+#
+# Candidate-side shape (same as aider/goose, OPPOSITE of the Half B evaluator):
+#   * /workspace WRITABLE (the harness edits files to produce a patch)
+#   * joins the `pollmevals-sandbox` INTERNAL net -> reaches ONLY the proxy.
+#
+# Pinned dependencies (Library-first; bump deliberately):
+#   node     22-slim      (opencode ships a Bun-compiled binary via npm
+#                          optionalDependencies, opencode-linux-<arch>)
+#   opencode-ai 1.15.13    (npm; pulls the platform binary)
+#   git      (Debian stable) -- opencode tracks edits; we capture the diff via git
+#
+# opencode headless gotchas:
+#   * opencode resolves custom AI-SDK provider packages (e.g.
+#     @ai-sdk/openai-compatible) at RUN time via npm -- which the no-egress
+#     sandbox cannot do. So we pre-install that provider at BUILD time and the
+#     recipe points opencode at it (or uses a built-in provider). The opencode.json
+#     config is written by the recipe (config_files), not baked (run-time proxy URL).
+#   * `opencode run -m <provider>/<model> --dangerously-skip-permissions "<task>"`
+#     is the one-shot headless form; `--print-logs`/`--format json` for output.
+#
+# Build:  docker build -t pollmevals-harness-opencode:0.1.0 infra/docker/harness-opencode/
+#         (or: make harness-image-opencode)
+# Smoke:  docker run --rm pollmevals-harness-opencode:0.1.0   # -> prints opencode version
+
+FROM node:22-slim AS base
+
+# git: opencode tracks edits; the launcher captures the diff host-side via git.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install opencode globally (pulls the platform binary via optionalDependencies).
+# The recipe uses opencode's BUILT-IN `openai` provider with a baseURL override
+# pointed at our proxy — that provider is bundled in the binary, so the no-egress
+# run-time container needs no npm at all (verified: a smoke run edited a file with
+# zero network beyond the proxy). Pinned.
+RUN npm install -g --no-fund --no-audit opencode-ai@1.15.13 \
+    && npm cache clean --force \
+    && opencode --version
+
+# Non-root user: defense-in-depth on top of --cap-drop=ALL and
+# --security-opt=no-new-privileges. The node base already ships a uid-1000 `node`
+# user, which matches the host bind owner so the produced patch is writable back.
+USER node
+WORKDIR /workspace
+
+# Default git identity so a fresh snapshot is a valid repo to diff against.
+RUN git config --global user.email "harness@pollmevals.local" \
+    && git config --global user.name "pollmevals-harness" \
+    && git config --global init.defaultBranch main \
+    && git config --global --add safe.directory /workspace
+
+# opencode writes telemetry/state under $HOME; keep it inside the container.
+ENV OPENCODE_DISABLE_AUTOUPDATE=1
+
+# Default command is a harmless version probe; DockerHarnessLauncher overrides
+# `command` with the full `opencode run ...` invocation.
+CMD ["opencode", "--version"]

stacks/opencode/stack.yaml

@@ -4,13 +4,13 @@ name: OpenCode
 base_model_slug: configurable
 agent_cli: opencode
 
-# Proven headless-via-proxy 2026-06-02 (memory: research-cli-harness-execution).
-# Model-agnostic. Config opencode.json:
-#   provider.litellm = { npm: "@ai-sdk/openai-compatible",
-#     options: { baseURL: "http://localhost:4000/v1",
-#                apiKey: "{env:LITELLM_MASTER_KEY}" },
-#     models: { "<MODEL>": { name: "<MODEL>" } } }
-# Run: opencode run "<prompt>" -m litellm/<MODEL>
+# Model-agnostic (Vercel AI SDK). Recipe codified + smoked 2026-06-03
+# (_opencode_invocation in stack_executor.py; memory: research-cli-harness-execution).
+# Uses opencode's BUILT-IN `openai` provider (bundled in the binary → no run-time
+# npm fetch, works on the no-egress sandbox) with a baseURL override to our proxy.
+# config_files writes opencode.json: provider.openai.options.baseURL=<proxy>/v1 +
+# models.{<MODEL>}; env OPENAI_API_KEY=$LITELLM_MASTER_KEY.
+# Run: opencode run -m openai/<MODEL> "<prompt>"
 
 layers:
   L0_bare_llm: false