diff --git a/Makefile b/Makefile index 3a490e6..ee65c36 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,8 @@ endif eval-core-test litellm-up litellm-down stack-up stack-down stack-status \ openrouter-smoke env-check \ harness-image-aider harness-image-goose harness-image-opencode \ - harness-image-crush harness-image-cline sandbox-net-up + harness-image-crush harness-image-cline \ + harness-image-pi harness-image-gptme harness-image-mini-swe sandbox-net-up demo-run: python -m pollmevals_eval_core.demo_run --tasks evals/tasks --output artifacts @@ -139,6 +140,15 @@ harness-image-crush: harness-image-cline: docker build -t pollmevals-harness-cline:0.1.0 infra/docker/harness-cline/ +# Build the pi / gptme / mini-SWE harness images (RFC-006 Phase 5). No spend. +# pi = node+npm (native tool_calls); gptme + mini-swe = python+pip (text-tolerant). +harness-image-pi: + docker build -t pollmevals-harness-pi:0.1.0 infra/docker/harness-pi/ +harness-image-gptme: + docker build -t pollmevals-harness-gptme:0.1.0 infra/docker/harness-gptme/ +harness-image-mini-swe: + docker build -t pollmevals-harness-mini-swe:0.1.0 infra/docker/harness-mini-swe/ + # Build the Python eval sandbox image (Phase 5 — PythonCorrectnessEvaluator, # runs BigCodeBench unittest suites). No spend — pulls base + pip scientific stack. eval-image-py: diff --git a/apps/eval-core-py/scripts/build_real_board.py b/apps/eval-core-py/scripts/build_real_board.py index 56369f2..5a4149d 100644 --- a/apps/eval-core-py/scripts/build_real_board.py +++ b/apps/eval-core-py/scripts/build_real_board.py @@ -94,6 +94,14 @@ # Cline: model-agnostic; same coder models (qwen-3-14b is too weak for its # tool-use — a real compat data point; the stronger coders work). _CLINE_MODELS = ["qwen-3-14b", "qwen3-coder-30b", "codestral", "devstral"] +# pi needs NATIVE tool_calls (openai-completions) → only models whose proxy +# backend emits them (qwen3-coder-30b returns text-format → pi no-ops). Verified +# native-tool on the proxy: devstral, codestral, qwen3-235b, glm-4-32b. +_PI_MODELS = ["devstral", "codestral", "qwen3-235b", "glm-4-32b"] +# gptme + mini-SWE are text-tolerant (gptme strips local/; mini uses +# litellm_textbased fences) → run the same 4 coders for comparison. +_GPTME_MODELS = ["qwen-3-14b", "qwen3-coder-30b", "codestral", "devstral"] +_MINI_MODELS = ["qwen-3-14b", "qwen3-coder-30b", "codestral", "devstral"] # Per-stack candidate model lists for --add-stack (merge ONE harness column in). _STACK_MODELS = { "aider": _AIDER_MODELS, @@ -101,6 +109,9 @@ "opencode": _OPENCODE_MODELS, "crush": _CRUSH_MODELS, "cline": _CLINE_MODELS, + "pi": _PI_MODELS, + "gptme": _GPTME_MODELS, + "mini-swe": _MINI_MODELS, } _SEEDS = [1, 2] _TASK = "be_01_jwt_auth" diff --git a/apps/eval-core-py/src/orchestrator/stack_executor.py b/apps/eval-core-py/src/orchestrator/stack_executor.py index 1b901ad..fcdb5a9 100644 --- a/apps/eval-core-py/src/orchestrator/stack_executor.py +++ b/apps/eval-core-py/src/orchestrator/stack_executor.py @@ -384,15 +384,148 @@ def _cline_invocation( ) +def _pi_invocation( + proxy_base_url: str, api_key: str, model_alias: str, prompt: str +) -> ProxyInvocation: + """pi (@earendil-works/pi-coding-agent) recipe — PROVEN (2026-06-03 smoke). + + Minimal model-agnostic coding agent (read/write/edit/bash). Config rides a + models.json that pi finds via ``PI_CODING_AGENT_DIR`` (default ~/.pi/agent — + NOT /workspace), so we point that env at /workspace/.pi/agent and write the + file there via config_files (the launcher mkdir -p's the parent). api_key is a + literal ``$LITELLM_MASTER_KEY`` pi expands from env (never written to a file). + ``api: openai-completions`` = native OpenAI chat-completions (tool-calling) -> + only run pi on models that emit NATIVE tool_calls on the proxy (devstral / + codestral / qwen3-235b / glm-4-32b); qwen3-coder-30b returns text-format tool + calls its backend doesn't parse and pi silently no-ops (a real compat finding). + """ + base = proxy_base_url.rstrip("/") + models_json = json.dumps( + { + "providers": { + "litellm": { + "baseUrl": f"{base}/v1", + "api": "openai-completions", + "apiKey": "$LITELLM_MASTER_KEY", + "models": [ + { + "id": model_alias, + "name": model_alias, + "reasoning": False, + "input": ["text"], + "contextWindow": 32768, + "maxTokens": 8192, + "cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0}, + } + ], + } + } + } + ) + return ProxyInvocation( + env={"LITELLM_MASTER_KEY": api_key, "PI_CODING_AGENT_DIR": "/workspace/.pi/agent"}, + config_files={".pi/agent/models.json": models_json}, + extra_args=["--model", f"litellm/{model_alias}", "--no-context-files", "-p"], + prompt_args=[prompt], + ) + + +def _gptme_invocation( + proxy_base_url: str, api_key: str, model_alias: str, prompt: str +) -> ProxyInvocation: + """gptme recipe — PROVEN (2026-06-03 smoke). ENV-ONLY (no config file). + + gptme reads a custom OpenAI-compatible endpoint from ``OPENAI_BASE_URL`` and + selects the model as ``local/`` (the ``local/`` prefix is stripped before + the call -> the proxy gets the bare alias). ``-n`` is non-interactive (implies + --no-confirm -> tools auto-execute); ``-w .`` pins the workspace to the cwd + (/workspace). gptme has NO turn cap and may loop "verifying" after writing - + the wall-clock timeout bounds it; the patch is written BEFORE the verify loop, + so a timeout-kill still yields a valid patch. The tiktoken cache is baked into + the image (the no-egress sandbox can't download encodings). + """ + base = proxy_base_url.rstrip("/") + return ProxyInvocation( + env={ + "OPENAI_BASE_URL": f"{base}/v1", + "OPENAI_API_KEY": api_key, + "MODEL": f"local/{model_alias}", + "GPTME_LOGS_HOME": "/home/harness/.local/share/gptme/logs", + }, + config_files={}, + extra_args=["-n", "-w", "."], + prompt_args=["-m", f"local/{model_alias}", prompt], + ) + + +# mini-SWE-agent's built-in config files (re-added first because ANY -c REPLACES +# the default config; inline -c specs then merge onto it). +_MINISWE_CFG = "/usr/local/lib/python3.12/site-packages/minisweagent/config" + + +def _mini_swe_invocation( + proxy_base_url: str, api_key: str, model_alias: str, prompt: str +) -> ProxyInvocation: + """mini-SWE-agent recipe — PROVEN (2026-06-03 smoke). The "bare validator loop". + + The 100-line minimal SWE loop (reason -> bash -> observe -> iterate), LiteLLM- + backed. ``--environment-class local`` runs bash directly in /workspace (NEVER a + nested Docker). Two env guards are load-bearing: ``MSWEA_CONFIGURED=true`` + (skip the interactive setup wizard -> else it hangs on stdin) and + ``MSWEA_COST_TRACKING=ignore_errors`` (proxy reports $0 -> else a RuntimeError + on the first response). ``litellm_textbased`` parses ```` ```mswea_bash_command ```` + fences instead of native tool_calls -> works with open coders (qwen3-coder-30b) + whose proxy backend lacks a tool-call parser. The trajectory goes to /tmp (out + of /workspace) so it never pollutes the captured patch. + """ + base = proxy_base_url.rstrip("/") + return ProxyInvocation( + env={ + "MSWEA_CONFIGURED": "true", + "MSWEA_COST_TRACKING": "ignore_errors", + "MSWEA_MODEL_NAME": f"openai/{model_alias}", + "OPENAI_API_BASE": f"{base}/v1", + "OPENAI_API_KEY": api_key, + }, + config_files={}, + extra_args=[ + "--environment-class", + "local", # bash in /workspace, NEVER a nested Docker + "-y", # yolo: no confirmation + "--exit-immediately", # no end-of-run prompt + "--model-class", + "litellm_textbased", + "-c", + f"{_MINISWE_CFG}/mini_textbased.yaml", # re-add built-in (any -c drops the default) + "-c", + f"model.model_name=openai/{model_alias}", + "-c", + f"model.model_kwargs.api_base={base}/v1", + "-c", + "model.model_kwargs.custom_llm_provider=openai", + "-c", + "agent.step_limit=40", # built-in default is 0 = unbounded + "-c", + "environment.timeout=600", + "-o", + "/tmp/mini-trajectory.json", # out of /workspace -> not in the patch + ], + prompt_args=["-t", prompt], + ) + + # Proven recipes (validated end-to-end via the proxy). aider is the RFC-006 first -# slice; goose / opencode / crush / cline (all 2026-06-03) are model-agnostic peers -# that run the same coder models for a clean harness comparison. +# slice; goose / opencode / crush / cline / pi / gptme / mini-swe (all 2026-06-03) +# are model-agnostic peers for the harness x model compat matrix. _PROVEN_RECIPES: dict[str, _RecipeBuilder] = { "aider": _aider_invocation, "goose": _goose_invocation, "opencode": _opencode_invocation, "crush": _crush_invocation, "cline": _cline_invocation, + "pi": _pi_invocation, + "gptme": _gptme_invocation, + "mini-swe": _mini_swe_invocation, } # Known harnesses whose recipe is proven in spikes but lands at its per-stack @@ -403,7 +536,6 @@ def _cline_invocation( "codex", "openhands", "hermes", - "pi", "forgeplan-framework", } ) diff --git a/apps/eval-core-py/tests/test_stack_executor.py b/apps/eval-core-py/tests/test_stack_executor.py index 0b59c2f..ef8297d 100644 --- a/apps/eval-core-py/tests/test_stack_executor.py +++ b/apps/eval-core-py/tests/test_stack_executor.py @@ -295,8 +295,60 @@ def test_cline_recipe_is_proven(self) -> None: assert "cline --yolo 'do the thing'" in wrapper assert wrapper.rstrip().endswith("|| true") # exit masked; the patch decides - def test_supported_harnesses_are_aider_goose_opencode_crush_cline(self) -> None: - assert supported_harnesses() == frozenset({"aider", "goose", "opencode", "crush", "cline"}) + def test_pi_recipe_is_proven(self) -> None: + inv = build_proxy_invocation( + "pi", + proxy_base_url="http://pollmevals-litellm-proxy:4000", + api_key="sk-local-xyz", + model_alias="devstral", + prompt="do the thing", + ) + assert inv.env["LITELLM_MASTER_KEY"] == "sk-local-xyz" + assert inv.env["PI_CODING_AGENT_DIR"] == "/workspace/.pi/agent" + assert inv.extra_args == ["--model", "litellm/devstral", "--no-context-files", "-p"] + assert inv.prompt_args == ["do the thing"] + cfg = json.loads(inv.config_files[".pi/agent/models.json"]) + prov = cfg["providers"]["litellm"] + assert prov["baseUrl"] == "http://pollmevals-litellm-proxy:4000/v1" + assert prov["apiKey"] == "$LITELLM_MASTER_KEY" # literal; key not in file + assert prov["models"][0]["id"] == "devstral" + + def test_gptme_recipe_is_proven(self) -> None: + inv = build_proxy_invocation( + "gptme", + proxy_base_url="http://pollmevals-litellm-proxy:4000/", + api_key="sk-local-xyz", + model_alias="qwen3-coder-30b", + prompt="do the thing", + ) + assert inv.env["OPENAI_BASE_URL"] == "http://pollmevals-litellm-proxy:4000/v1" + assert inv.env["OPENAI_API_KEY"] == "sk-local-xyz" + assert inv.env["MODEL"] == "local/qwen3-coder-30b" + assert inv.config_files == {} + assert inv.extra_args == ["-n", "-w", "."] + assert inv.prompt_args == ["-m", "local/qwen3-coder-30b", "do the thing"] + + def test_mini_swe_recipe_is_proven(self) -> None: + inv = build_proxy_invocation( + "mini-swe", + proxy_base_url="http://pollmevals-litellm-proxy:4000", + api_key="sk-local-xyz", + model_alias="qwen3-coder-30b", + prompt="do the thing", + ) + assert inv.env["MSWEA_CONFIGURED"] == "true" + assert inv.env["MSWEA_COST_TRACKING"] == "ignore_errors" + assert inv.env["MSWEA_MODEL_NAME"] == "openai/qwen3-coder-30b" + assert inv.env["OPENAI_API_BASE"] == "http://pollmevals-litellm-proxy:4000/v1" + assert "--environment-class" in inv.extra_args + assert "local" in inv.extra_args # never a nested Docker + assert "litellm_textbased" in inv.extra_args + assert inv.prompt_args == ["-t", "do the thing"] + + def test_supported_harnesses_are_all_eight(self) -> None: + assert supported_harnesses() == frozenset( + {"aider", "goose", "opencode", "crush", "cline", "pi", "gptme", "mini-swe"} + ) # --------------------------------------------------------------------------- diff --git a/apps/site/public/board.json b/apps/site/public/board.json index b624079..3468b57 100644 --- a/apps/site/public/board.json +++ b/apps/site/public/board.json @@ -58,6 +58,37 @@ "L2_tools" ], "family": "agnostic" + }, + { + "stack_id": "pi", + "name": "PI", + "level": 2, + "layers": [ + "L1_system_prompt", + "L2_tools" + ], + "family": "vendor" + }, + { + "stack_id": "gptme", + "name": "gptme", + "level": 4, + "layers": [ + "L1_system_prompt", + "L2_tools", + "L4_file_memory" + ], + "family": "agnostic" + }, + { + "stack_id": "mini-swe", + "name": "mini-SWE-agent", + "level": 7, + "layers": [ + "L2_tools", + "L7_validator" + ], + "family": "agnostic" } ], "models": [ @@ -926,6 +957,264 @@ "type_safety": 4.5 }, "on_frontier": false + }, + { + "model_id": "codestral", + "stack_id": "pi", + "mean_score": 6.67, + "mean_cost_usd": 0.0, + "mean_latency_ms": 9099, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 6.67, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 7.25, + "correctness": 7.25, + "error_handling": 6.5, + "security_posture": 7.5, + "test_alignment": 6.5, + "type_safety": 5.0 + }, + "on_frontier": false + }, + { + "model_id": "devstral", + "stack_id": "pi", + "mean_score": 7.46, + "mean_cost_usd": 0.0, + "mean_latency_ms": 80619, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 7.46, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 8.5, + "correctness": 7.75, + "error_handling": 7.0, + "security_posture": 8.0, + "test_alignment": 7.5, + "type_safety": 6.0 + }, + "on_frontier": false + }, + { + "model_id": "glm-4-32b", + "stack_id": "pi", + "mean_score": null, + "mean_cost_usd": 0.0, + "mean_latency_ms": 2889, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": null, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": {}, + "on_frontier": false + }, + { + "model_id": "qwen3-235b", + "stack_id": "pi", + "mean_score": 6.17, + "mean_cost_usd": 0.0, + "mean_latency_ms": 34632, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 6.17, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 7.0, + "correctness": 5.5, + "error_handling": 6.5, + "security_posture": 8.0, + "test_alignment": 6.0, + "type_safety": 4.0 + }, + "on_frontier": false + }, + { + "model_id": "codestral", + "stack_id": "gptme", + "mean_score": 7.17, + "mean_cost_usd": 0.0, + "mean_latency_ms": 90187, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 7.17, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 7.0, + "correctness": 8.5, + "error_handling": 7.5, + "security_posture": 8.0, + "test_alignment": 6.5, + "type_safety": 5.5 + }, + "on_frontier": false + }, + { + "model_id": "devstral", + "stack_id": "gptme", + "mean_score": null, + "mean_cost_usd": 0.0, + "mean_latency_ms": 478462, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": null, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": {}, + "on_frontier": false + }, + { + "model_id": "qwen-3-14b", + "stack_id": "gptme", + "mean_score": null, + "mean_cost_usd": 0.0, + "mean_latency_ms": 343815, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": null, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": {}, + "on_frontier": false + }, + { + "model_id": "qwen3-coder-30b", + "stack_id": "gptme", + "mean_score": null, + "mean_cost_usd": 0.0, + "mean_latency_ms": 29991, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": null, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": {}, + "on_frontier": false + }, + { + "model_id": "codestral", + "stack_id": "mini-swe", + "mean_score": null, + "mean_cost_usd": 0.0, + "mean_latency_ms": 262096, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": null, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": {}, + "on_frontier": false + }, + { + "model_id": "devstral", + "stack_id": "mini-swe", + "mean_score": null, + "mean_cost_usd": 0.0, + "mean_latency_ms": 262841, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": null, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": {}, + "on_frontier": false + }, + { + "model_id": "qwen-3-14b", + "stack_id": "mini-swe", + "mean_score": 6.54, + "mean_cost_usd": 0.0, + "mean_latency_ms": 65330, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 6.54, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 6.75, + "correctness": 6.75, + "error_handling": 7.25, + "security_posture": 7.75, + "test_alignment": 6.5, + "type_safety": 4.25 + }, + "on_frontier": false + }, + { + "model_id": "qwen3-coder-30b", + "stack_id": "mini-swe", + "mean_score": 7.08, + "mean_cost_usd": 0.0, + "mean_latency_ms": 60591, + "pass_hat_k": null, + "quality_per_dollar": null, + "per_task": { + "be_01_jwt_auth": { + "score": 7.08, + "cost_usd": 0.0, + "pass_hat_k": null + } + }, + "per_criterion": { + "code_clarity": 7.0, + "correctness": 8.0, + "error_handling": 8.0, + "security_posture": 9.0, + "test_alignment": 7.0, + "type_safety": 3.5 + }, + "on_frontier": false } ] } diff --git a/infra/docker/harness-gptme/Dockerfile b/infra/docker/harness-gptme/Dockerfile new file mode 100644 index 0000000..ae012e0 --- /dev/null +++ b/infra/docker/harness-gptme/Dockerfile @@ -0,0 +1,102 @@ +# syntax=docker/dockerfile:1.7 +# +# pollmevals-harness-gptme -- sandboxed gptme CLI image (RFC-006 Half A, +# candidate side). gptme is a model-AGNOSTIC terminal agent (it speaks the +# OpenAI-compatible Chat Completions API), so it runs the SAME open coder models +# aider/goose/opencode do -> another clean "swap the harness, hold the model" +# comparison column. +# +# Same candidate-side shape as the other harnesses (the OPPOSITE of the Half B +# evaluator image): +# * /workspace is WRITABLE (the harness edits files to produce a patch) +# * the container joins the `pollmevals-sandbox` INTERNAL network so it can +# reach ONLY the LiteLLM proxy (bastion) -- no general egress. gptme talks +# to the proxy (which meters every token); it never holds the upstream key. +# +# Pinned dependencies (Library-first; bump deliberately, never floating): +# python 3.12-slim (gptme is a pure-Python package; reuses the cached +# aider/goose/eval-py base layer) +# gptme 0.31.0 (latest on PyPI 2026-06-03) +# git (Debian stable) -- gptme tracks edits; we capture the diff via git +# +# gptme headless / config gotchas (ENV-ONLY config, no config file needed): +# * gptme reads provider config from env: OPENAI_BASE_URL + OPENAI_API_KEY +# select the OpenAI-compatible endpoint (our proxy). The MODEL env (or the +# -m flag) takes a `local/` form -- gptme strips the `local/` prefix +# and forwards the bare model name to the proxy, which is exactly the model +# id LiteLLM exposes (e.g. qwen3-coder-30b). All injected by the recipe at +# run time (proxy URL + key + model), NOT baked here. +# * GPTME_LOGS_HOME points gptme's conversation logs inside the container HOME +# so a fresh, possibly read-only-elsewhere bind never blocks log writes. +# * gptme counts tokens with `tiktoken`, which LAZILY DOWNLOADS its encoding +# file (cl100k_base / o200k_base) from openaipublic.blob.core.windows.net on +# first use. The no-egress sandbox blocks that DNS, so an un-warmed image +# fatals BEFORE it ever reaches the proxy (empirical: smoke iteration 1 died +# with "Failed to resolve openaipublic.blob.core.windows.net"). Fix: warm +# both base encodings into a baked TIKTOKEN_CACHE_DIR at BUILD time (build +# has egress; only the run-time container is sealed). tiktoken keys each +# blob by sha1(url), so a populated cache dir = a pure offline hit, zero net. +# * `gptme -n ...` is non-interactive and implies --no-confirm, so the file/ +# shell tools auto-execute (a headless run must never wait on a prompt); +# `-w .` pins the workspace to cwd (/workspace). +# +# Build: +# docker build -t pollmevals-harness-gptme:0.1.0 infra/docker/harness-gptme/ +# (or: make harness-image-gptme) +# +# Run (referenced from DockerHarnessLauncher -- not invoked by humans directly): +# docker run --rm \ +# --network=pollmevals-sandbox \ +# --cap-drop=ALL --security-opt=no-new-privileges:true \ +# --memory=2g --pids-limit=256 \ +# -v $(pwd)/snapshot:/workspace \ +# -e OPENAI_BASE_URL=http://pollmevals-litellm-proxy:4000/v1 \ +# -e OPENAI_API_KEY=$LITELLM_MASTER_KEY \ +# -e MODEL=local/ \ +# pollmevals-harness-gptme:0.1.0 \ +# gptme -n -w . -m local/ "" +# +# Smoke (no proxy needed): +# docker run --rm pollmevals-harness-gptme:0.1.0 # -> prints gptme version + +FROM python:3.12-slim AS base + +# git: gptme tracks edits; the launcher captures the diff host-side via git. +# ca-certificates: TLS to the proxy bastion. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Pinned gptme (library-first; bump deliberately). Installed at build time so +# the run-time container needs no PyPI access on the internal network. +RUN pip install --no-cache-dir gptme==0.31.0 + +# Warm tiktoken's encoding cache at BUILD time so the no-egress run-time +# container never tries to download cl100k_base/o200k_base from Azure blob (see +# header). World-readable dir so the non-root `harness` user can read it. +ENV TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache +RUN mkdir -p "$TIKTOKEN_CACHE_DIR" \ + && python3 -c "import tiktoken; [tiktoken.get_encoding(e) for e in ('cl100k_base','o200k_base')]" \ + && chmod -R a+rX "$TIKTOKEN_CACHE_DIR" + +# Non-root user: defense-in-depth on top of --cap-drop=ALL and +# --security-opt=no-new-privileges. The python base ships NO uid-1000 user, so +# we create one; uid 1000 matches the host bind owner so the produced patch is +# writable back on the host snapshot. +RUN useradd --create-home --uid 1000 harness +USER harness +WORKDIR /workspace + +# Default git identity so gptme's edit-tracking never fails on a fresh snapshot. +RUN git config --global user.email "harness@pollmevals.local" \ + && git config --global user.name "pollmevals-harness" \ + && git config --global init.defaultBranch main \ + && git config --global --add safe.directory /workspace + +# Keep gptme's conversation logs inside the container HOME (see header). +ENV GPTME_LOGS_HOME=/home/harness/.local/share/gptme/logs + +# Default command is a harmless version probe; DockerHarnessLauncher overrides +# `command` with the full `gptme -n ...` invocation. No restrictive ENTRYPOINT +# so the launcher can pass either an argv list or an explicit sh -c wrapper. +CMD ["gptme", "--version"] diff --git a/infra/docker/harness-mini-swe/Dockerfile b/infra/docker/harness-mini-swe/Dockerfile new file mode 100644 index 0000000..7639d19 --- /dev/null +++ b/infra/docker/harness-mini-swe/Dockerfile @@ -0,0 +1,98 @@ +# syntax=docker/dockerfile:1.7 +# +# pollmevals-harness-mini-swe -- sandboxed "mini-swe-agent" (CLI `mini`) image +# (RFC-006 Half A, candidate side). mini-swe-agent is a deliberately minimal, +# model-AGNOSTIC SWE agent (just bash, no bespoke tools) -- it runs the SAME +# coder models aider/goose do, so it's another clean "swap the harness, hold the +# model" comparison column, but at the OTHER extreme of the scaffolding ladder +# (a single bash tool + a built-in step/validator loop, no tool zoo). +# +# Same candidate-side shape as harness-aider / harness-goose (the OPPOSITE of the +# Half B evaluator image): +# * /workspace is WRITABLE (the harness edits files to produce a patch) +# * the container joins the `pollmevals-sandbox` INTERNAL network so it can +# reach ONLY the LiteLLM proxy (bastion) -- no general egress. mini talks to +# the proxy (which meters every token); it never holds the upstream key. +# +# Pinned dependencies (Library-first; bump deliberately, never floating): +# python 3.12-slim (mini-swe-agent supports 3.10+) +# mini-swe-agent 2.3.0 (PyPI; provides the `mini` console script) +# git (Debian stable) -- the launcher captures the diff host-side +# via git; mini itself runs bash edits in-place +# +# Headless gotchas baked in (each one hangs or crashes a non-interactive run): +# * MSWEA_CONFIGURED=true -- skip the first-run interactive setup wizard +# (otherwise `mini` blocks asking for config). +# * MSWEA_COST_TRACKING=ignore_errors +# -- our LiteLLM proxy reports $0 cost for local +# models; mini's default cost tracking raises +# a RuntimeError on a $0 / missing-cost reply. +# `ignore_errors` downgrades that to a warning. +# The model name + proxy wiring (MSWEA_MODEL_NAME / OPENAI_API_BASE / +# OPENAI_API_KEY, and the inline `-c model.*` overrides) are injected at RUN +# time by the recipe (_mini_swe_invocation in stack_executor.py), not baked +# here -- the image stays model-agnostic. +# +# Build: +# docker build -t pollmevals-harness-mini-swe:0.1.0 infra/docker/harness-mini-swe/ +# (or: make harness-image-mini-swe) +# +# Run (referenced from DockerHarnessLauncher -- not invoked by humans directly): +# docker run --rm \ +# --network=pollmevals-sandbox \ +# --cap-drop=ALL --security-opt=no-new-privileges:true \ +# --memory=2g --pids-limit=256 \ +# -v $(pwd)/snapshot:/workspace \ +# -e MSWEA_CONFIGURED=true \ +# -e MSWEA_COST_TRACKING=ignore_errors \ +# -e MSWEA_MODEL_NAME=openai/ \ +# -e OPENAI_API_BASE=http://pollmevals-litellm-proxy:4000/v1 \ +# -e OPENAI_API_KEY=$LITELLM_MASTER_KEY \ +# pollmevals-harness-mini-swe:0.1.0 \ +# mini --environment-class local -y --exit-immediately \ +# -c model.model_name=openai/ \ +# -c model.model_kwargs.api_base=http://pollmevals-litellm-proxy:4000/v1 \ +# -c model.model_kwargs.custom_llm_provider=openai \ +# -c agent.step_limit=40 -c environment.timeout=600 \ +# -t "" +# (--environment-class local is CRITICAL: it runs bash in /workspace, NOT a +# nested Docker -- the sandbox has no docker socket and must never get one.) +# +# Smoke (no proxy needed): +# docker run --rm pollmevals-harness-mini-swe:0.1.0 # -> prints `mini --help` + +FROM python:3.12-slim AS base + +# git: the launcher tracks edits / computes the diff via git host-side, and a +# fresh snapshot must be a valid repo. ca-certificates: TLS to the proxy. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Pinned mini-swe-agent (library-first; bump deliberately). Installed at build +# time so the run-time container needs no PyPI access on the internal network. +# Provides the `mini` console script. +RUN pip install --no-cache-dir mini-swe-agent==2.3.0 + +# Non-root user: defense-in-depth on top of --cap-drop=ALL and +# --security-opt=no-new-privileges. uid 1000 matches the host bind owner so the +# produced patch is writable back on the host snapshot. +RUN useradd --create-home --uid 1000 harness +USER harness +WORKDIR /workspace + +# Default git identity so a fresh snapshot is a valid repo to diff against. +RUN git config --global user.email "harness@pollmevals.local" \ + && git config --global user.name "pollmevals-harness" \ + && git config --global init.defaultBranch main \ + && git config --global --add safe.directory /workspace + +# Headless defaults (see header). Skip the setup wizard and tolerate the proxy's +# $0 cost reports. Model + proxy wiring are injected by the recipe at run time. +ENV MSWEA_CONFIGURED=true \ + MSWEA_COST_TRACKING=ignore_errors + +# Default command is a harmless help probe; DockerHarnessLauncher overrides +# `command` with the full `mini ...` invocation. No restrictive ENTRYPOINT so the +# launcher can pass either an argv list or an explicit sh -c wrapper. +CMD ["mini", "--help"] diff --git a/infra/docker/harness-pi/Dockerfile b/infra/docker/harness-pi/Dockerfile new file mode 100644 index 0000000..868ff03 --- /dev/null +++ b/infra/docker/harness-pi/Dockerfile @@ -0,0 +1,81 @@ +# syntax=docker/dockerfile:1.7 +# +# pollmevals-harness-pi -- sandboxed pi-coding-agent CLI image (RFC-006 Half A). +# Model-AGNOSTIC agentic CLI (`@earendil-works/pi-coding-agent`, pure JS, binary +# `pi`) -> runs the SAME open coder models as aider/opencode/goose -- another +# clean "swap the harness, hold the model" column. +# +# Candidate-side shape (same as aider/opencode/goose, OPPOSITE of the Half B +# evaluator image): +# * /workspace is WRITABLE (the harness edits files to produce a patch) +# * joins the `pollmevals-sandbox` INTERNAL net -> reaches ONLY the LiteLLM +# proxy (bastion). pi talks to the proxy (which meters every token); it +# never holds the upstream provider key (RFC-006 invariant: no un-metered +# egress). +# +# Pinned dependencies (Library-first; bump deliberately, never floating): +# node 22-slim (pi requires Node >=22.19; the 22-slim tag ships a +# 22.19+ runtime AND a pre-made uid-1000 `node` user we +# reuse -- matches the host bind owner so the patch is +# writable back, same as the opencode harness) +# @earendil-works/pi-coding-agent 0.78.0 (npm; pure JS, no platform binary) +# git (Debian stable) -- pi tracks edits; we capture the diff via git +# +# pi headless gotchas baked in (researched + source-verified): +# * pi reads its models.json from $PI_CODING_AGENT_DIR (default ~/.pi/agent). +# We default that env to /workspace/.pi/agent so a config the launcher writes +# INTO the bind-mounted /workspace is found at run time. The models.json +# itself is written by the recipe (config_files) -- not baked -- because it +# carries the run-time proxy URL + the chosen model. pi expands +# $LITELLM_MASTER_KEY (and other $VARS) in models.json from the env. +# * PI_OFFLINE / PI_SKIP_VERSION_CHECK / PI_TELEMETRY=0 -- the no-egress +# sandbox can't reach the registry/telemetry endpoints; without these pi +# would block or warn on a network call it can never complete. +# * Run form: `pi --model litellm/ --no-context-files -p ""`. +# No permission gate in the headless `-p` path (writes unconditionally), so +# a one-shot run never stalls on an interactive confirmation. +# +# Build: docker build -t pollmevals-harness-pi:0.1.0 infra/docker/harness-pi/ +# (or: make harness-image-pi) +# Smoke: docker run --rm pollmevals-harness-pi:0.1.0 # -> prints pi version + +FROM node:22-slim AS base + +# git: pi tracks edits; the launcher captures the diff host-side via git. +# ca-certificates: TLS to the proxy bastion. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Install pi globally (pure-JS npm package -- no optional platform binary). +# Pulled at BUILD time so the no-egress run-time container needs no npm at all. +# Pinned (library-first; bump deliberately). +RUN npm install -g --no-fund --no-audit @earendil-works/pi-coding-agent@0.78.0 \ + && npm cache clean --force \ + && pi --version + +# Non-root user: defense-in-depth on top of --cap-drop=ALL and +# --security-opt=no-new-privileges. The node base already ships a uid-1000 +# `node` user, which matches the host bind owner so the produced patch is +# writable back. Reuse it (do NOT useradd). +USER node +WORKDIR /workspace + +# Default git identity so a fresh snapshot is a valid repo to diff against. +RUN git config --global user.email "harness@pollmevals.local" \ + && git config --global user.name "pollmevals-harness" \ + && git config --global init.defaultBranch main \ + && git config --global --add safe.directory /workspace + +# Headless / no-egress defaults (see header). PI_CODING_AGENT_DIR points pi at a +# config dir inside the bind-mounted workspace so the recipe-written models.json +# is found. The proxy URL + model live in that models.json (run-time), not here. +ENV PI_CODING_AGENT_DIR=/workspace/.pi/agent \ + PI_OFFLINE=1 \ + PI_SKIP_VERSION_CHECK=1 \ + PI_TELEMETRY=0 + +# Default command is a harmless version probe; DockerHarnessLauncher overrides +# `command` with the full `pi --model ... -p ...` invocation. No restrictive +# ENTRYPOINT so the launcher can pass either an argv list or an `sh -c` wrapper. +CMD ["pi", "--version"] diff --git a/stacks/gptme/stack.yaml b/stacks/gptme/stack.yaml new file mode 100644 index 0000000..52b06f4 --- /dev/null +++ b/stacks/gptme/stack.yaml @@ -0,0 +1,63 @@ +schema_version: pollmevals.stack.v1 +slug: gptme +name: gptme +base_model_slug: configurable +agent_cli: gptme + +# gptme (gptme.org): a terminal chat-agent (L1 system prompt) with built-in +# shell + save/patch file tools (L2) that reads project context files such as +# gptme.toml / referenced files (L4). Model-agnostic — speaks the OpenAI +# Chat-Completions API, so it runs the same open coder models as aider/goose/ +# opencode; the board orders harnesses by max true layer. +# +# ENV-ONLY config (no config file) — recipe codified + smoked end-to-end +# 2026-06-03 (memory: research-cli-harness-execution): +# env OPENAI_BASE_URL=http://:4000/v1 +# OPENAI_API_KEY=$LITELLM_MASTER_KEY +# MODEL=local/ # gptme strips `local/` → proxy gets bare +# GPTME_LOGS_HOME=/home/harness/.local/share/gptme/logs +# TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache # baked in image (offline tokenizer) +# run gptme -n -w . -m local/ "" +# -n non-interactive (implies --no-confirm → tools auto-execute) +# -w . pin workspace to cwd (/workspace) +# prompt is the trailing positional arg. +# +# Loop note: gptme has NO --max-turns/step cap (verified). Some open models +# (e.g. qwen3-coder-30b) re-`cat` the file to "verify" and don't self-terminate, +# so the run is bounded ONLY by max_wall_clock_seconds (executor `timeout` +# wrapper). The patch is already on disk before the loop, so a timeout-bounded +# exit still yields a valid patch. + +layers: + L0_bare_llm: false + L1_system_prompt: true + L2_tools: true + L3_skills: false + L4_file_memory: true # reads gptme.toml / referenced project context files + L5_vector_memory: false + L6_subagents: false + L7_validator: false + L8_framework: null +execution: + mode: repository_patch + command: gptme + args: [] # -n -w . + -m local/ + prompt are built in _gptme_invocation +input_contract: + receives: + - task_prompt + - repository_snapshot + - allowed_files +output_contract: + produces: + - final_answer + - patch + - trace +limits: + max_wall_clock_seconds: 600 + max_tool_calls: 50 + max_input_tokens: 50000 + max_output_tokens: 10000 +sandbox: + network: false + writable_paths: + - /workspace diff --git a/stacks/mini-swe/stack.yaml b/stacks/mini-swe/stack.yaml new file mode 100644 index 0000000..8eb3550 --- /dev/null +++ b/stacks/mini-swe/stack.yaml @@ -0,0 +1,50 @@ +schema_version: pollmevals.stack.v1 +slug: mini-swe +name: mini-SWE-agent +base_model_slug: configurable +agent_cli: mini-swe +# mini-SWE-agent (SWE-agent/mini-swe-agent): the ~100-line minimal SWE loop — a +# bare model (L0) + a single bash tool (L2) inside a reason→run→observe→iterate +# validator loop (L7). No persona/skills/memory/subagents. Recipe codified + +# smoked 2026-06-03 (_mini_swe_invocation; memory: research-cli-harness-execution): +# env MSWEA_CONFIGURED=true (skip wizard) + MSWEA_COST_TRACKING=ignore_errors +# (proxy $0 → else RuntimeError) + MSWEA_MODEL_NAME=openai/ + +# OPENAI_API_BASE=/v1 + OPENAI_API_KEY. +# run: mini --environment-class local (bash in /workspace, NO nested Docker) -y +# --exit-immediately --model-class litellm_textbased (open coders emit +# fenced bash, not native tool_calls) -c /mini_textbased.yaml -c +# model.* -c agent.step_limit=40 -c environment.timeout=600 -o /tmp/... -t "

" +# (all flags built in the recipe; command/args here are minimal). +layers: + L0_bare_llm: false + L1_system_prompt: false + L2_tools: true + L3_skills: false + L4_file_memory: false + L5_vector_memory: false + L6_subagents: false + L7_validator: true + L8_framework: null +execution: + mode: repository_patch + command: mini + args: [] # all flags + prompt are built in _mini_swe_invocation +input_contract: + receives: + - task_prompt + - repository_snapshot + - allowed_files +output_contract: + produces: + - final_answer + - patch + - trace +limits: + max_wall_clock_seconds: 600 + max_tool_calls: 50 + max_input_tokens: 50000 + max_output_tokens: 10000 +sandbox: + network: false + writable_paths: + - /workspace diff --git a/stacks/pi/stack.yaml b/stacks/pi/stack.yaml index 5f8972a..585e74c 100644 --- a/stacks/pi/stack.yaml +++ b/stacks/pi/stack.yaml @@ -1,14 +1,22 @@ schema_version: pollmevals.stack.v1 slug: pi -name: Pi Terminal Coding Harness +name: PI base_model_slug: configurable agent_cli: pi +# pi (@earendil-works/pi-coding-agent): minimal agent (L1 system prompt) with +# read/write/edit/bash tools (L2). Recipe codified + smoked 2026-06-03 +# (_pi_invocation; memory: research-cli-harness-execution). config rides +# .pi/agent/models.json via config_files; pi finds it via PI_CODING_AGENT_DIR= +# /workspace/.pi/agent. Run: pi --model litellm/ --no-context-files -p "

". +# `--no-context-files` disables L3 skills + L4 context-memory for a clean agnostic +# column. api:openai-completions = native tool_calls → run only on models that emit +# them on the proxy (devstral/codestral/qwen3-235b/glm-4-32b; NOT qwen3-coder-30b). layers: L0_bare_llm: false L1_system_prompt: true L2_tools: true - L3_skills: true - L4_file_memory: true + L3_skills: false + L4_file_memory: false L5_vector_memory: false L6_subagents: false L7_validator: false