From 94740c4151020608920b7d9f093443e751c50848 Mon Sep 17 00:00:00 2001 From: gogocat Date: Tue, 2 Jun 2026 18:20:01 +0300 Subject: [PATCH 1/7] =?UTF-8?q?feat(executor):=20StackExecutor=20Phase=201?= =?UTF-8?q?=20=E2=80=94=20adapter=20+=20proxy=20recipe=20+=20launcher=20se?= =?UTF-8?q?am=20(RFC-006)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Half A candidate path (run agent-CLI harness -> patch), no Docker / no spend. - StackAdapter: operational parse of stacks//stack.yaml (distinct from the manifest StackPin run-pin — separate concern, separate contract) - build_proxy_invocation: per-CLI proxy recipe. aider proven (the RFC-006 first slice); other known harnesses surface HarnessRecipePending (recipe lands at its Phase-5 per-stack smoke); unknown CLI -> UnsupportedHarnessError - HarnessLauncher Protocol seam + deterministic FakeHarnessLauncher; DockerHarnessLauncher is a typed Phase-2 stub (launch() raises) - build_docker_run_kwargs: writable /workspace bind + locked-down flags (cap_drop ALL, no-new-privileges). PROXY_ONLY egress bridge raises NetworkPolicyNotConfigured — the security-sensitive bridge is an open RFC-006 decision wired in Phase 2 - StackExecutor.execute: adapter + recipe -> HarnessRunPlan -> launch -> patch/trace/cost(metered)/status; never drops a result (FR-009 discipline) Gates: ruff + mypy --strict clean; 33 new unit tests; full suite 690 passed. Refs: rfc-006 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/orchestrator/stack_executor.py | 669 ++++++++++++++++++ .../eval-core-py/tests/test_stack_executor.py | 389 ++++++++++ 2 files changed, 1058 insertions(+) create mode 100644 apps/eval-core-py/src/orchestrator/stack_executor.py create mode 100644 apps/eval-core-py/tests/test_stack_executor.py diff --git a/apps/eval-core-py/src/orchestrator/stack_executor.py b/apps/eval-core-py/src/orchestrator/stack_executor.py new file mode 100644 index 0000000..a47c613 --- /dev/null +++ b/apps/eval-core-py/src/orchestrator/stack_executor.py @@ -0,0 +1,669 @@ +"""Stack executor — Half A candidate path: run an agent-CLI harness in a sandbox. + +RFC-006 Phase 1 (no Docker, mocked). Provides the core executor + the per-CLI +proxy-config builder + patch/trace/cost capture behind a Protocol seam, so unit +tests run without Docker or network. NO spend. + +Where this sits in the pipeline (two-half sandbox, RFC-006): + + Half A — candidate (THIS module): run the harness CLI -> produce a patch. + Half B — evaluator (evaluators/sandbox/runner.py): run the produced code + -> scores. Already built. + +The executor produces the patch that Half B then scores. + +Design discipline (mirrors existing seams): + * EvalCaller / FakeEvalCaller (eval_caller.py) — the Protocol-seam pattern: + a real implementation + a deterministic Fake for unit tests. Here the seam + is ``HarnessLauncher`` (Docker is behind it), so ``StackExecutor`` is fully + testable with ``FakeHarnessLauncher`` — no daemon, no network, no spend. + * SandboxRun (evaluators/sandbox/runner.py) — docker-py + frozen security + policy. Half B is ``network=none`` + ``read_only=True``; Half A is the + opposite (writable /workspace + a single allowed egress to the LiteLLM + proxy), so it CANNOT reuse SandboxRun — hence a distinct launcher seam. + +Scope split: + * ``raw-llm`` (execution.mode = direct_completion) stays on InspectEvalCaller. + * Every ``repository_patch`` stack (aider, claude-code, codex, goose, + openhands, opencode, hermes, ...) runs through StackExecutor. + +Phase boundary: the real ``DockerHarnessLauncher`` (network bridge + image run) +lands in RFC-006 Phase 2. Phase 1 ships the Protocol, the deterministic Fake, +the pure docker-kwargs builder (tested), and the full executor orchestration. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import UTC, datetime +from decimal import Decimal +from enum import StrEnum +from pathlib import Path +from typing import Protocol, runtime_checkable + +import yaml +from pydantic import BaseModel, ConfigDict, Field + +from src.orchestrator.cost import PricingTuple, compute_cost + +logger = logging.getLogger(__name__) + +# Default LiteLLM proxy endpoint (Wave 1 infra). All harness model calls +# traverse this so we (a) choose the model and (b) meter every token. +DEFAULT_PROXY_BASE_URL = "http://localhost:4000" + + +# --------------------------------------------------------------------------- +# Exceptions +# --------------------------------------------------------------------------- + + +class StackExecutorError(Exception): + """Base class for all stack-executor errors.""" + + +class UnsupportedHarnessError(StackExecutorError): + """Raised when an ``agent_cli`` has no known proxy recipe at all.""" + + +class HarnessRecipePending(StackExecutorError): + """Raised for a KNOWN harness whose proxy recipe is not yet codified. + + The recipe is proven (memory: research-cli-harness-execution) but is + validated + landed at its per-stack smoke (RFC-006 Phase 5). Surfacing this + distinctly from UnsupportedHarnessError keeps "we haven't wired it yet" from + masquerading as "we don't support it". + """ + + +class NetworkPolicyNotConfigured(StackExecutorError): + """Raised when the PROXY_ONLY network bridge is requested before Phase 2. + + The proxy-only egress bridge (the RFC-006 "crux") is a security-sensitive + open decision (sidecar-in-container-net vs host-gateway firewall rule). It + is intentionally NOT baked into the kwargs builder until that decision is + made — this exception is the single, explicit seam where it plugs in. + """ + + +# --------------------------------------------------------------------------- +# StackAdapter — executor-side parse of stacks//stack.yaml +# --------------------------------------------------------------------------- +# +# Distinct from contracts.StackPin (the write-once run-pin: stack_id + sha256). +# This is the OPERATIONAL view the executor needs. ``extra="ignore"`` because +# stack.yaml carries many keys this module does not consume (schema_version, +# layers, input_contract, output_contract); we model only execution/limits/ +# sandbox and let the rest pass through untouched. + + +class ExecutionMode(StrEnum): + """How a stack produces its output.""" + + DIRECT_COMPLETION = "direct_completion" # raw-llm -> InspectEvalCaller + REPOSITORY_PATCH = "repository_patch" # CLI harness -> StackExecutor + + +class ExecutionSpec(BaseModel): + """``execution:`` block of stack.yaml.""" + + model_config = ConfigDict(extra="ignore", frozen=True) + + mode: ExecutionMode + command: str + args: list[str] = Field(default_factory=list) + + +class LimitsSpec(BaseModel): + """``limits:`` block — wall-clock / tool-call / token caps.""" + + model_config = ConfigDict(extra="ignore", frozen=True) + + max_wall_clock_seconds: int = 300 + max_tool_calls: int = 0 + max_input_tokens: int = 50_000 + max_output_tokens: int = 10_000 + + +class SandboxSpec(BaseModel): + """``sandbox:`` block — isolation hints.""" + + model_config = ConfigDict(extra="ignore", frozen=True) + + network: bool = False + writable_paths: list[str] = Field(default_factory=lambda: ["/workspace"]) + + +class StackAdapter(BaseModel): + """Operational parse of one ``stacks//stack.yaml`` adapter.""" + + model_config = ConfigDict(extra="ignore", frozen=True) + + slug: str + name: str = "" + agent_cli: str | None = None + base_model_slug: str = "configurable" + execution: ExecutionSpec + limits: LimitsSpec = Field(default_factory=LimitsSpec) + sandbox: SandboxSpec = Field(default_factory=SandboxSpec) + + @classmethod + def from_yaml_text(cls, text: str) -> StackAdapter: + """Parse adapter YAML text into a StackAdapter (validates the schema).""" + data = yaml.safe_load(text) + if not isinstance(data, dict): + raise ValueError("stack.yaml must parse to a mapping") + return cls.model_validate(data) + + @classmethod + def from_yaml_path(cls, path: Path) -> StackAdapter: + """Load + parse ``stacks//stack.yaml`` from disk.""" + return cls.from_yaml_text(Path(path).read_text(encoding="utf-8")) + + +# --------------------------------------------------------------------------- +# Per-CLI proxy recipe — how one harness is pointed at the proxy + given a task +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ProxyInvocation: + """Everything needed to point one harness CLI at the LiteLLM proxy. + + Fields: + env: Env vars injected into the sandbox (proxy base + key). + config_files: Relative-path -> file-content map written into the + sandbox before the run (e.g. codex config.toml). Empty + for env-only CLIs like aider. + extra_args: Args appended to ``execution.command + execution.args`` to + select the model / provider (e.g. ``--model openai/``). + prompt_args: Args that deliver the task prompt (e.g. ``--message

``). + """ + + env: dict[str, str] + config_files: dict[str, str] + extra_args: list[str] + prompt_args: list[str] + + +# A recipe builder takes (proxy_base_url, api_key, model_alias, prompt). +_RecipeBuilder = Callable[[str, str, str, str], ProxyInvocation] + + +def _aider_invocation( + proxy_base_url: str, api_key: str, model_alias: str, prompt: str +) -> ProxyInvocation: + """Aider recipe — PROVEN first slice (2026-06-02 spike; memory). + + aider is model-agnostic: it reads ``OPENAI_API_BASE`` / ``OPENAI_API_KEY`` + and takes the model as ``--model openai/``. The task is delivered + non-interactively via ``--message`` (stack.yaml already pins + ``--no-pretty --yes``). + """ + base = proxy_base_url.rstrip("/") + return ProxyInvocation( + env={ + "OPENAI_API_BASE": f"{base}/v1", + "OPENAI_API_KEY": api_key, + }, + config_files={}, + extra_args=["--model", f"openai/{model_alias}"], + prompt_args=["--message", prompt], + ) + + +# Proven recipes (validated end-to-end via the proxy). aider is the RFC-006 +# first slice (aider x qwen x be_01). +_PROVEN_RECIPES: dict[str, _RecipeBuilder] = { + "aider": _aider_invocation, +} + +# Known harnesses whose recipe is proven in spikes but lands at its per-stack +# smoke (RFC-006 Phase 5). Keying off the stack.yaml ``agent_cli`` value. +_PENDING_RECIPES: frozenset[str] = frozenset( + { + "claude-code", + "codex", + "opencode", + "goose", + "openhands", + "hermes", + "cline", + "pi", + "forgeplan-framework", + } +) + + +def build_proxy_invocation( + agent_cli: str | None, + *, + proxy_base_url: str, + api_key: str, + model_alias: str, + prompt: str, +) -> ProxyInvocation: + """Resolve the proxy invocation for ``agent_cli``. + + Raises: + UnsupportedHarnessError: ``agent_cli`` is None or completely unknown. + HarnessRecipePending: known harness, recipe lands at its Phase-5 smoke. + """ + if agent_cli is None: + raise UnsupportedHarnessError("stack has no agent_cli (is it a raw-llm stack?)") + builder = _PROVEN_RECIPES.get(agent_cli) + if builder is not None: + return builder(proxy_base_url, api_key, model_alias, prompt) + if agent_cli in _PENDING_RECIPES: + raise HarnessRecipePending( + f"harness '{agent_cli}' recipe is proven but not yet codified " + "(RFC-006 Phase 5 per-stack smoke); only 'aider' is wired in Phase 1" + ) + raise UnsupportedHarnessError(f"unknown agent_cli '{agent_cli}'") + + +def supported_harnesses() -> frozenset[str]: + """Return the set of agent_cli values wired in this phase (proven only).""" + return frozenset(_PROVEN_RECIPES) + + +# --------------------------------------------------------------------------- +# Launcher seam — Docker is behind this Protocol (testable without a daemon) +# --------------------------------------------------------------------------- + + +class NetworkPolicy(StrEnum): + """Sandbox egress policy for a harness run.""" + + NONE = "none" # fully isolated (Half B evaluator default) + PROXY_ONLY = "proxy_only" # the crux: allow ONLY the LiteLLM proxy host:port + + +@dataclass(frozen=True) +class HarnessRunPlan: + """Fully-resolved instructions for one sandboxed harness run.""" + + image: str + command: list[str] + workdir: str + mount_dir: Path # writable bind at /workspace + environment: dict[str, str] + config_files: dict[str, str] + timeout_s: int + network_policy: NetworkPolicy + proxy_host: str + proxy_port: int + + +@dataclass(frozen=True) +class HarnessRunOutcome: + """Raw outcome of one launcher run (pre-scoring).""" + + exit_code: int + patch: str # unified git diff captured from /workspace ("" if none) + trace: str # harness stdout / structured trace blob + stderr: str + input_tokens: int + output_tokens: int + tool_calls: int + wall_ms: int + timed_out: bool + + +@runtime_checkable +class HarnessLauncher(Protocol): + """Protocol for running one harness plan in a sandbox. + + Implementors: + * DockerHarnessLauncher — real path (RFC-006 Phase 2). + * FakeHarnessLauncher — deterministic mock for StackExecutor unit tests. + """ + + async def launch(self, plan: HarnessRunPlan) -> HarnessRunOutcome: ... + + +def build_docker_run_kwargs(plan: HarnessRunPlan) -> dict[str, object]: + """Translate a HarnessRunPlan into docker-py ``containers.run`` kwargs. + + Pure + tested (the security-sensitive surface is worth pinning now even + though ``launch()`` is Phase 2). Mirrors SandboxRun's frozen flags but for + the candidate side: the workspace bind is **writable** (the harness must + produce a patch) while keep-the-rest-locked-down still holds (cap_drop ALL, + no-new-privileges, pids/mem limits). + + The network branch is the explicit Phase-2 seam: + * NONE -> ``network_mode="none"`` (fully isolated). + * PROXY_ONLY -> raises NetworkPolicyNotConfigured until the proxy-only + bridge decision is made (RFC-006 crux). + """ + kwargs: dict[str, object] = { + "image": plan.image, + "command": plan.command, + "detach": True, + "working_dir": plan.workdir, + # ---- Security policy (candidate side) ---- + "read_only": False, # harness must write to /workspace to produce a patch + "cap_drop": ["ALL"], + "security_opt": ["no-new-privileges:true"], + "mem_limit": "2g", # harness CLIs are heavier than Half B evaluators + "nano_cpus": 2_000_000_000, # 2.0 CPU + "pids_limit": 256, + # ---- Writable workspace bind (the produced patch lives here) ---- + "volumes": {str(plan.mount_dir): {"bind": "/workspace", "mode": "rw"}}, + "environment": dict(plan.environment), + } + if plan.network_policy is NetworkPolicy.NONE: + kwargs["network_mode"] = "none" + else: + # PROXY_ONLY — the security-sensitive bridge is an open RFC-006 decision. + raise NetworkPolicyNotConfigured( + "PROXY_ONLY egress bridge is an open RFC-006 decision " + "(sidecar-in-container-net vs host-gateway firewall). Wire it in " + f"Phase 2 for proxy {plan.proxy_host}:{plan.proxy_port}." + ) + return kwargs + + +class DockerHarnessLauncher: + """Real HarnessLauncher — RFC-006 Phase 2 (NOT wired in Phase 1). + + Present as the typed seam + provenance anchor; ``launch()`` raises until + Phase 2 builds the CLI sandbox images and resolves the proxy-only bridge. + The pure ``build_docker_run_kwargs`` above is already testable. + """ + + async def launch(self, plan: HarnessRunPlan) -> HarnessRunOutcome: + raise NotImplementedError( + "DockerHarnessLauncher.launch is RFC-006 Phase 2 — needs the CLI " + "sandbox image + the proxy-only network bridge. Phase 1 uses " + "FakeHarnessLauncher." + ) + + +@dataclass +class FakeHarnessLauncher: + """Deterministic mock HarnessLauncher for StackExecutor unit tests. + + Captures the last plan it was handed (so tests can assert on the assembled + command / env / network policy) and returns a configurable outcome. Never + touches Docker or the network. + """ + + outcome: HarnessRunOutcome = field( + default_factory=lambda: HarnessRunOutcome( + exit_code=0, + patch="diff --git a/x b/x\n+ok\n", + trace="fake-trace", + stderr="", + input_tokens=1000, + output_tokens=500, + tool_calls=3, + wall_ms=4000, + timed_out=False, + ) + ) + last_plan: HarnessRunPlan | None = None + + async def launch(self, plan: HarnessRunPlan) -> HarnessRunOutcome: + self.last_plan = plan + return self.outcome + + +# --------------------------------------------------------------------------- +# StackExecutor — orchestration: adapter + proxy recipe -> plan -> result +# --------------------------------------------------------------------------- + + +class ExecStatus(StrEnum): + """Terminal status of one stack execution.""" + + OK = "ok" # patch produced, exit 0 + NO_PATCH = "no_patch" # ran clean but produced an empty diff + TIMEOUT = "timeout" # hit the wall-clock limit + FAILED = "failed" # non-zero exit / launcher error + UNSUPPORTED = "unsupported" # wrong mode / no recipe for this harness + + +def default_image_for_cli(agent_cli: str) -> str: + """Default sandbox image name for a harness (built in RFC-006 Phase 2).""" + return f"pollmevals-harness-{agent_cli}:0.1.0" + + +@dataclass(frozen=True) +class StackExecRequest: + """All inputs to execute one (model x stack x task x seed) harness run. + + Mirrors EvalRequest (eval_caller.py) but carries the resolved StackAdapter + + the task prompt + a writable repo snapshot dir (the harness mutates it). + + Fields: + model_id: Cost-attribution key (provider route, matches the + pricing_snapshot key, e.g. "openrouter/qwen/qwen-3-14b"). + model_alias: Proxy-facing alias the CLI sends (litellm model_name). + """ + + eval_id: str + model_id: str + model_alias: str + stack: StackAdapter + task_id: str + task_prompt: str + repo_snapshot_dir: Path + seed: int + timeout_s: int = 600 + + +@dataclass(frozen=True) +class StackExecResult: + """Outcome of one StackExecutor.execute() invocation.""" + + request: StackExecRequest + status: ExecStatus + patch: str | None + trace: str + cost_usd: Decimal + input_tokens: int + output_tokens: int + tool_calls: int + wall_ms: int + error_detail: str | None + started_at: datetime + completed_at: datetime + + +@dataclass +class _TokenStats: + """Minimal EvalStatsLike for compute_cost (needs only token counts). + + Not frozen: the EvalStatsLike protocol declares ``input_tokens`` / + ``output_tokens`` as settable variables, which a frozen dataclass would + expose as read-only (mypy --strict rejects the mismatch). + """ + + input_tokens: int + output_tokens: int + + +class StackExecutor: + """Run a ``repository_patch`` stack: adapter + proxy recipe -> patch + cost. + + Args: + launcher: any HarnessLauncher (DockerHarnessLauncher / FakeHarnessLauncher). + proxy_base_url: LiteLLM proxy base (harness model calls route here). + api_key: proxy auth (LITELLM_MASTER_KEY — NOT the upstream provider key). + pricing_snapshot: model_id -> PricingTuple, frozen at run start + (RFC-001 Invariant #3). Cost is computed from proxy-metered tokens. + image_resolver: agent_cli -> image name (injectable for tests). + """ + + def __init__( + self, + *, + launcher: HarnessLauncher, + proxy_base_url: str = DEFAULT_PROXY_BASE_URL, + api_key: str = "", + pricing_snapshot: dict[str, PricingTuple] | None = None, + image_resolver: Callable[[str], str] = default_image_for_cli, + ) -> None: + self._launcher = launcher + self._proxy_base_url = proxy_base_url.rstrip("/") + self._api_key = api_key + self._pricing_snapshot = pricing_snapshot or {} + self._image_resolver = image_resolver + + def _proxy_host_port(self) -> tuple[str, int]: + """Split the proxy base URL into (host, port) for the network bridge.""" + netloc = self._proxy_base_url.split("://", 1)[-1].split("/", 1)[0] + host, _, port = netloc.partition(":") + return host or "localhost", int(port) if port.isdigit() else 80 + + async def execute(self, request: StackExecRequest) -> StackExecResult: + """Execute one harness run and map its outcome to a StackExecResult. + + Never raises on a harness/launcher failure — failures are captured in + ``status`` + ``error_detail`` (FR-009 discipline: a result is always + produced). Re-raises nothing the launcher emits except programmer errors. + """ + started_at = datetime.now(UTC) + agent_cli = request.stack.agent_cli + + # Guard 1: this executor only handles repository_patch stacks. + if request.stack.execution.mode is not ExecutionMode.REPOSITORY_PATCH: + return self._unsupported( + request, + started_at, + f"mode '{request.stack.execution.mode.value}' is not " + "repository_patch (raw-llm runs through InspectEvalCaller)", + ) + + # Guard 2: resolve the proxy recipe (unsupported / pending -> UNSUPPORTED). + try: + invocation = build_proxy_invocation( + agent_cli, + proxy_base_url=self._proxy_base_url, + api_key=self._api_key, + model_alias=request.model_alias, + prompt=request.task_prompt, + ) + except (UnsupportedHarnessError, HarnessRecipePending) as exc: + return self._unsupported(request, started_at, str(exc)) + + # agent_cli is guaranteed non-None here (build_proxy_invocation guards it). + assert agent_cli is not None + + # Assemble the full command: base command + adapter args + model + prompt. + command = [ + request.stack.execution.command, + *request.stack.execution.args, + *invocation.extra_args, + *invocation.prompt_args, + ] + host, port = self._proxy_host_port() + plan = HarnessRunPlan( + image=self._image_resolver(agent_cli), + command=command, + workdir="/workspace", + mount_dir=request.repo_snapshot_dir, + environment=invocation.env, + config_files=invocation.config_files, + timeout_s=min(request.timeout_s, request.stack.limits.max_wall_clock_seconds), + network_policy=NetworkPolicy.PROXY_ONLY, + proxy_host=host, + proxy_port=port, + ) + + # Launch (the only place that can fail with an environment error). + try: + outcome = await self._launcher.launch(plan) + except Exception as exc: # graceful: capture, never drop (FR-009) + logger.exception("Harness launch failed for eval_id=%s", request.eval_id) + return self._failed(request, started_at, f"launcher error: {type(exc).__name__}: {exc}") + + return self._from_outcome(request, started_at, outcome) + + # ------------------------------------------------------------------ + # Outcome -> result mapping + # ------------------------------------------------------------------ + + def _cost(self, request: StackExecRequest, outcome: HarnessRunOutcome) -> Decimal: + """Cost from proxy-metered tokens x the pinned pricing for this model.""" + pricing = self._pricing_snapshot.get(request.model_id) + if pricing is None: + return Decimal("0") + return compute_cost( + _TokenStats(outcome.input_tokens, outcome.output_tokens), + pricing, + ) + + def _from_outcome( + self, + request: StackExecRequest, + started_at: datetime, + outcome: HarnessRunOutcome, + ) -> StackExecResult: + if outcome.timed_out: + status = ExecStatus.TIMEOUT + error_detail: str | None = f"timed out after {request.timeout_s}s" + patch: str | None = outcome.patch or None + elif outcome.exit_code != 0: + status = ExecStatus.FAILED + error_detail = f"harness exit_code={outcome.exit_code}" + patch = outcome.patch or None + elif not outcome.patch.strip(): + status = ExecStatus.NO_PATCH + error_detail = "harness produced an empty diff" + patch = None + else: + status = ExecStatus.OK + error_detail = None + patch = outcome.patch + + return StackExecResult( + request=request, + status=status, + patch=patch, + trace=outcome.trace, + cost_usd=self._cost(request, outcome), + input_tokens=outcome.input_tokens, + output_tokens=outcome.output_tokens, + tool_calls=outcome.tool_calls, + wall_ms=outcome.wall_ms, + error_detail=error_detail, + started_at=started_at, + completed_at=datetime.now(UTC), + ) + + def _unsupported( + self, request: StackExecRequest, started_at: datetime, detail: str + ) -> StackExecResult: + return self._terminal(request, started_at, ExecStatus.UNSUPPORTED, detail) + + def _failed( + self, request: StackExecRequest, started_at: datetime, detail: str + ) -> StackExecResult: + return self._terminal(request, started_at, ExecStatus.FAILED, detail) + + def _terminal( + self, + request: StackExecRequest, + started_at: datetime, + status: ExecStatus, + detail: str, + ) -> StackExecResult: + """Build a zero-cost result for a pre-launch terminal condition.""" + return StackExecResult( + request=request, + status=status, + patch=None, + trace="", + cost_usd=Decimal("0"), + input_tokens=0, + output_tokens=0, + tool_calls=0, + wall_ms=0, + error_detail=detail, + started_at=started_at, + completed_at=datetime.now(UTC), + ) diff --git a/apps/eval-core-py/tests/test_stack_executor.py b/apps/eval-core-py/tests/test_stack_executor.py new file mode 100644 index 0000000..6d2c038 --- /dev/null +++ b/apps/eval-core-py/tests/test_stack_executor.py @@ -0,0 +1,389 @@ +"""Unit tests for the RFC-006 Phase 1 stack executor (Half A candidate path). + +No Docker, no network, no spend — everything runs through FakeHarnessLauncher +and the pure builders. Mirrors the test_eval_caller.py class-grouping style. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path + +import pytest + +from src.orchestrator.cost import PricingTuple +from src.orchestrator.stack_executor import ( + DEFAULT_PROXY_BASE_URL, + DockerHarnessLauncher, + ExecStatus, + ExecutionMode, + FakeHarnessLauncher, + HarnessRecipePending, + HarnessRunOutcome, + HarnessRunPlan, + NetworkPolicy, + NetworkPolicyNotConfigured, + StackAdapter, + StackExecRequest, + StackExecutor, + UnsupportedHarnessError, + build_docker_run_kwargs, + build_proxy_invocation, + default_image_for_cli, + supported_harnesses, +) + +# --------------------------------------------------------------------------- +# Fixtures / helpers +# --------------------------------------------------------------------------- + +_AIDER_YAML = """ +schema_version: pollmevals.stack.v1 +slug: aider +name: Aider +base_model_slug: configurable +agent_cli: aider +layers: + L0_bare_llm: false + L2_tools: true +execution: + mode: repository_patch + command: aider + args: + - --no-pretty + - --yes +input_contract: + receives: [task_prompt, repository_snapshot] +output_contract: + produces: [patch, trace] +limits: + max_wall_clock_seconds: 600 + max_tool_calls: 50 + max_input_tokens: 50000 + max_output_tokens: 10000 +sandbox: + network: false + writable_paths: + - /workspace +""" + +_RAW_LLM_YAML = """ +schema_version: pollmevals.stack.v1 +slug: raw-llm +name: Raw LLM +agent_cli: null +execution: + mode: direct_completion + command: litellm + args: [] +limits: + max_wall_clock_seconds: 300 +""" + + +def _aider_adapter() -> StackAdapter: + return StackAdapter.from_yaml_text(_AIDER_YAML) + + +def _request(stack: StackAdapter, tmp_path: Path, **overrides: object) -> StackExecRequest: + defaults: dict[str, object] = { + "eval_id": "abc123", + "model_id": "openrouter/qwen/qwen-3-14b", + "model_alias": "qwen-3-14b", + "stack": stack, + "task_id": "be_01_jwt_auth", + "task_prompt": "Implement JWT auth middleware.", + "repo_snapshot_dir": tmp_path, + "seed": 1, + } + defaults.update(overrides) + return StackExecRequest(**defaults) # type: ignore[arg-type] + + +_PRICING = { + "openrouter/qwen/qwen-3-14b": PricingTuple( + "openrouter/qwen/qwen-3-14b", + Decimal("0.20"), + Decimal("0.80"), + datetime(2026, 1, 1, tzinfo=UTC), + ) +} + + +# --------------------------------------------------------------------------- +# StackAdapter parsing +# --------------------------------------------------------------------------- + + +class TestStackAdapter: + def test_parses_aider_adapter(self) -> None: + a = _aider_adapter() + assert a.slug == "aider" + assert a.agent_cli == "aider" + assert a.execution.mode is ExecutionMode.REPOSITORY_PATCH + assert a.execution.command == "aider" + assert a.execution.args == ["--no-pretty", "--yes"] + assert a.limits.max_wall_clock_seconds == 600 + assert a.sandbox.network is False + + def test_ignores_unknown_keys(self) -> None: + # schema_version / layers / *_contract are not modelled; must not raise. + a = _aider_adapter() + assert a.name == "Aider" + + def test_raw_llm_mode_and_null_cli(self) -> None: + a = StackAdapter.from_yaml_text(_RAW_LLM_YAML) + assert a.execution.mode is ExecutionMode.DIRECT_COMPLETION + assert a.agent_cli is None + + def test_from_yaml_path(self, tmp_path: Path) -> None: + p = tmp_path / "stack.yaml" + p.write_text(_AIDER_YAML, encoding="utf-8") + a = StackAdapter.from_yaml_path(p) + assert a.slug == "aider" + + def test_non_mapping_yaml_raises(self) -> None: + with pytest.raises(ValueError, match="must parse to a mapping"): + StackAdapter.from_yaml_text("- just\n- a\n- list\n") + + def test_limits_and_sandbox_defaults(self) -> None: + a = StackAdapter.from_yaml_text( + "slug: x\nexecution:\n mode: repository_patch\n command: x\n" + ) + assert a.limits.max_wall_clock_seconds == 300 + assert a.sandbox.writable_paths == ["/workspace"] + + +# --------------------------------------------------------------------------- +# Proxy recipe builder +# --------------------------------------------------------------------------- + + +class TestProxyInvocation: + def test_aider_recipe_is_proven(self) -> None: + inv = build_proxy_invocation( + "aider", + proxy_base_url="http://localhost:4000", + api_key="sk-local-xyz", + model_alias="qwen-3-14b", + prompt="do the thing", + ) + assert inv.env["OPENAI_API_BASE"] == "http://localhost:4000/v1" + assert inv.env["OPENAI_API_KEY"] == "sk-local-xyz" + assert inv.extra_args == ["--model", "openai/qwen-3-14b"] + assert inv.prompt_args == ["--message", "do the thing"] + assert inv.config_files == {} + + def test_aider_strips_trailing_slash(self) -> None: + inv = build_proxy_invocation( + "aider", + proxy_base_url="http://localhost:4000/", + api_key="k", + model_alias="m", + prompt="p", + ) + assert inv.env["OPENAI_API_BASE"] == "http://localhost:4000/v1" + + @pytest.mark.parametrize("cli", ["claude-code", "codex", "goose", "openhands", "opencode"]) + def test_known_but_pending_harness_raises_pending(self, cli: str) -> None: + with pytest.raises(HarnessRecipePending, match="Phase 5"): + build_proxy_invocation( + cli, proxy_base_url="x", api_key="k", model_alias="m", prompt="p" + ) + + def test_unknown_harness_raises_unsupported(self) -> None: + with pytest.raises(UnsupportedHarnessError, match="unknown agent_cli"): + build_proxy_invocation( + "nope", proxy_base_url="x", api_key="k", model_alias="m", prompt="p" + ) + + def test_none_cli_raises_unsupported(self) -> None: + with pytest.raises(UnsupportedHarnessError, match="no agent_cli"): + build_proxy_invocation( + None, proxy_base_url="x", api_key="k", model_alias="m", prompt="p" + ) + + def test_supported_harnesses_is_aider_only_in_phase_1(self) -> None: + assert supported_harnesses() == frozenset({"aider"}) + + +# --------------------------------------------------------------------------- +# Docker kwargs builder (pure, security-sensitive surface) +# --------------------------------------------------------------------------- + + +def _plan(policy: NetworkPolicy, tmp_path: Path) -> HarnessRunPlan: + return HarnessRunPlan( + image="img:0.1.0", + command=["aider", "--yes"], + workdir="/workspace", + mount_dir=tmp_path, + environment={"OPENAI_API_KEY": "k"}, + config_files={}, + timeout_s=600, + network_policy=policy, + proxy_host="localhost", + proxy_port=4000, + ) + + +class TestDockerRunKwargs: + def test_workspace_bind_is_writable(self, tmp_path: Path) -> None: + kw = build_docker_run_kwargs(_plan(NetworkPolicy.NONE, tmp_path)) + volumes = kw["volumes"] + assert isinstance(volumes, dict) + assert volumes[str(tmp_path)] == {"bind": "/workspace", "mode": "rw"} + assert kw["read_only"] is False + + def test_security_flags_locked_down(self, tmp_path: Path) -> None: + kw = build_docker_run_kwargs(_plan(NetworkPolicy.NONE, tmp_path)) + assert kw["cap_drop"] == ["ALL"] + assert kw["security_opt"] == ["no-new-privileges:true"] + assert kw["network_mode"] == "none" + + def test_proxy_only_is_an_open_decision(self, tmp_path: Path) -> None: + # The crux: the proxy-only egress bridge is not baked in until Phase 2. + with pytest.raises(NetworkPolicyNotConfigured, match="open RFC-006 decision"): + build_docker_run_kwargs(_plan(NetworkPolicy.PROXY_ONLY, tmp_path)) + + +# --------------------------------------------------------------------------- +# StackExecutor orchestration +# --------------------------------------------------------------------------- + + +class TestStackExecutorHappyPath: + @pytest.mark.asyncio + async def test_ok_status_and_patch(self, tmp_path: Path) -> None: + launcher = FakeHarnessLauncher() + ex = StackExecutor(launcher=launcher, pricing_snapshot=_PRICING) + result = await ex.execute(_request(_aider_adapter(), tmp_path)) + assert result.status is ExecStatus.OK + assert result.patch is not None and result.patch.startswith("diff --git") + assert result.error_detail is None + + @pytest.mark.asyncio + async def test_cost_computed_from_metered_tokens(self, tmp_path: Path) -> None: + # 1000 in * 0.20 + 500 out * 0.80 = 200 + 400 = 600 / 1e6 = 0.0006 + launcher = FakeHarnessLauncher() + ex = StackExecutor(launcher=launcher, pricing_snapshot=_PRICING) + result = await ex.execute(_request(_aider_adapter(), tmp_path)) + assert result.cost_usd == Decimal("0.000600") + assert result.input_tokens == 1000 + assert result.output_tokens == 500 + + @pytest.mark.asyncio + async def test_no_pricing_yields_zero_cost(self, tmp_path: Path) -> None: + ex = StackExecutor(launcher=FakeHarnessLauncher()) # no pricing_snapshot + result = await ex.execute(_request(_aider_adapter(), tmp_path)) + assert result.cost_usd == Decimal("0") + + @pytest.mark.asyncio + async def test_command_assembled_and_proxy_only(self, tmp_path: Path) -> None: + launcher = FakeHarnessLauncher() + ex = StackExecutor(launcher=launcher, pricing_snapshot=_PRICING) + await ex.execute(_request(_aider_adapter(), tmp_path, task_prompt="P")) + plan = launcher.last_plan + assert plan is not None + assert plan.command == [ + "aider", + "--no-pretty", + "--yes", + "--model", + "openai/qwen-3-14b", + "--message", + "P", + ] + assert plan.network_policy is NetworkPolicy.PROXY_ONLY + assert plan.mount_dir == tmp_path + assert plan.proxy_host == "localhost" + assert plan.proxy_port == 4000 + + @pytest.mark.asyncio + async def test_timeout_is_min_of_request_and_limit(self, tmp_path: Path) -> None: + launcher = FakeHarnessLauncher() + ex = StackExecutor(launcher=launcher, pricing_snapshot=_PRICING) + # request 9999 vs adapter limit 600 -> 600 + await ex.execute(_request(_aider_adapter(), tmp_path, timeout_s=9999)) + assert launcher.last_plan is not None + assert launcher.last_plan.timeout_s == 600 + + +class TestStackExecutorFailureModes: + @pytest.mark.asyncio + async def test_empty_patch_is_no_patch(self, tmp_path: Path) -> None: + launcher = FakeHarnessLauncher( + outcome=HarnessRunOutcome(0, " \n ", "t", "", 10, 5, 0, 100, False) + ) + ex = StackExecutor(launcher=launcher, pricing_snapshot=_PRICING) + result = await ex.execute(_request(_aider_adapter(), tmp_path)) + assert result.status is ExecStatus.NO_PATCH + assert result.patch is None + + @pytest.mark.asyncio + async def test_nonzero_exit_is_failed(self, tmp_path: Path) -> None: + launcher = FakeHarnessLauncher( + outcome=HarnessRunOutcome(1, "diff --git a b\n", "t", "boom", 10, 5, 0, 100, False) + ) + ex = StackExecutor(launcher=launcher) + result = await ex.execute(_request(_aider_adapter(), tmp_path)) + assert result.status is ExecStatus.FAILED + assert result.error_detail is not None and "exit_code=1" in result.error_detail + + @pytest.mark.asyncio + async def test_timed_out_is_timeout(self, tmp_path: Path) -> None: + launcher = FakeHarnessLauncher( + outcome=HarnessRunOutcome(137, "", "t", "", 10, 5, 0, 600_000, True) + ) + ex = StackExecutor(launcher=launcher) + result = await ex.execute(_request(_aider_adapter(), tmp_path)) + assert result.status is ExecStatus.TIMEOUT + + @pytest.mark.asyncio + async def test_direct_completion_is_unsupported(self, tmp_path: Path) -> None: + raw = StackAdapter.from_yaml_text(_RAW_LLM_YAML) + ex = StackExecutor(launcher=FakeHarnessLauncher()) + result = await ex.execute(_request(raw, tmp_path)) + assert result.status is ExecStatus.UNSUPPORTED + assert result.error_detail is not None and "repository_patch" in result.error_detail + + @pytest.mark.asyncio + async def test_pending_harness_is_unsupported_with_phase5_hint(self, tmp_path: Path) -> None: + codex_yaml = _AIDER_YAML.replace("slug: aider", "slug: codex").replace( + "agent_cli: aider", "agent_cli: codex" + ) + codex = StackAdapter.from_yaml_text(codex_yaml) + ex = StackExecutor(launcher=FakeHarnessLauncher()) + result = await ex.execute(_request(codex, tmp_path)) + assert result.status is ExecStatus.UNSUPPORTED + assert result.error_detail is not None and "Phase 5" in result.error_detail + + @pytest.mark.asyncio + async def test_launcher_exception_is_captured_not_raised(self, tmp_path: Path) -> None: + class Boom: + async def launch(self, plan: HarnessRunPlan) -> HarnessRunOutcome: + raise RuntimeError("daemon down") + + ex = StackExecutor(launcher=Boom()) + result = await ex.execute(_request(_aider_adapter(), tmp_path)) + assert result.status is ExecStatus.FAILED + assert result.error_detail is not None and "daemon down" in result.error_detail + + +# --------------------------------------------------------------------------- +# Phase-2 seam: Docker launcher is intentionally not wired yet +# --------------------------------------------------------------------------- + + +class TestPhase2Seam: + @pytest.mark.asyncio + async def test_docker_launcher_not_wired(self, tmp_path: Path) -> None: + with pytest.raises(NotImplementedError, match="Phase 2"): + await DockerHarnessLauncher().launch(_plan(NetworkPolicy.NONE, tmp_path)) + + def test_default_image_name(self) -> None: + assert default_image_for_cli("aider") == "pollmevals-harness-aider:0.1.0" + + def test_default_proxy_base_url(self) -> None: + assert DEFAULT_PROXY_BASE_URL == "http://localhost:4000" From 80058a5126c915428313a4dc5dfeeac3e99830ac Mon Sep 17 00:00:00 2001 From: gogocat Date: Tue, 2 Jun 2026 20:31:17 +0300 Subject: [PATCH 2/7] =?UTF-8?q?feat(executor):=20Phase=202=20=E2=80=94=20a?= =?UTF-8?q?ider=20sandbox=20image=20+=20bastion=20network=20(decision=20A)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC-006 Phase 2 (no spend): the image + network now exist; the real run is gated. - infra/docker/harness-aider/Dockerfile -> pollmevals-harness-aider:0.1.0 (python:3.12-slim + aider-chat 0.86.2 + git 2.47.3, non-root, writable /workspace). Built + smoke-verified (aider --version, git --version). - Network decision A wired declaratively: pollmevals-sandbox `internal: true` net in the litellm compose; the proxy joins it as the bastion (only reachable host, no external route -> harness reaches ONLY the metered proxy, and cap_drop ALL holds because no NET_ADMIN/iptables is needed). Portable Linux/macOS/CI. - stack_executor: PROXY_ONLY now joins the internal bastion net (no longer raises); in-sandbox the harness addresses the proxy by container name (SANDBOX_PROXY_BASE_URL), not localhost (the internal net has no host route); DockerHarnessLauncher.launch moves to Phase 3 (built + validated against the first real run, not blind). - Makefile: harness-image-aider (build) + sandbox-net-up (attach proxy to the bastion net on an already-running stack, idempotent). Gates: ruff + mypy --strict clean; 36 stack-executor tests; full suite 692 passed; compose validates with internal:true preserved. Refs: rfc-006 Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile | 20 +++++- .../src/orchestrator/stack_executor.py | 49 +++++++++---- .../eval-core-py/tests/test_stack_executor.py | 34 ++++++--- infra/docker-compose.litellm.yml | 15 ++++ infra/docker/harness-aider/Dockerfile | 71 +++++++++++++++++++ 5 files changed, 165 insertions(+), 24 deletions(-) create mode 100644 infra/docker/harness-aider/Dockerfile diff --git a/Makefile b/Makefile index c9e7d9a..4462c07 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,8 @@ endif validate-tasks validate-stacks reproduce \ smoke-run smoke-dry resume postmortem \ eval-core-test litellm-up litellm-down stack-up stack-down stack-status \ - openrouter-smoke env-check + openrouter-smoke env-check \ + harness-image-aider sandbox-net-up demo-run: python -m pollmevals_eval_core.demo_run --tasks evals/tasks --output artifacts @@ -109,6 +110,23 @@ stack-down: stack-status: @docker ps --filter 'name=pollmevals-' --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' +# --------------------------------------------------------------------------- +# Harness sandbox images (RFC-006 Half A — candidate-side agent CLIs) +# --------------------------------------------------------------------------- + +# Build the aider harness image (RFC-006 Phase 2). No spend — pulls base + pip. +harness-image-aider: + docker build -t pollmevals-harness-aider:0.1.0 infra/docker/harness-aider/ + +# Attach the proxy to the internal sandbox bastion net (RFC-006 decision A) on +# an ALREADY-running stack, without a full `stack-up` recreate. Idempotent. +sandbox-net-up: + @docker network inspect pollmevals-sandbox >/dev/null 2>&1 \ + || docker network create --internal pollmevals-sandbox + @docker network connect pollmevals-sandbox pollmevals-litellm-proxy 2>/dev/null \ + && echo "✅ proxy attached to pollmevals-sandbox" \ + || echo "ℹ️ proxy already on pollmevals-sandbox (or not running — 'make stack-up')" + # --------------------------------------------------------------------------- # Observability stack (LGTM + OTEL collector) — per NOTE-003 # --------------------------------------------------------------------------- diff --git a/apps/eval-core-py/src/orchestrator/stack_executor.py b/apps/eval-core-py/src/orchestrator/stack_executor.py index a47c613..5c9e7ac 100644 --- a/apps/eval-core-py/src/orchestrator/stack_executor.py +++ b/apps/eval-core-py/src/orchestrator/stack_executor.py @@ -54,6 +54,17 @@ # traverse this so we (a) choose the model and (b) meter every token. DEFAULT_PROXY_BASE_URL = "http://localhost:4000" +# RFC-006 network decision A (2026-06-02): the harness runs on a Docker +# `internal` network with the LiteLLM proxy attached as the only reachable host +# (bastion). The internal net has NO external route, so the harness reaches ONLY +# the proxy -- no NET_ADMIN needed, cap_drop ALL holds, portable Linux/macOS/CI. +SANDBOX_NETWORK = "pollmevals-sandbox" +PROXY_CONTAINER = "pollmevals-litellm-proxy" +# In-sandbox the proxy is addressed by its container name (Docker DNS), NOT +# localhost -- localhost:4000 only works from the host, and the internal net has +# no host route. This is what gets baked into the harness OPENAI_API_BASE. +SANDBOX_PROXY_BASE_URL = f"http://{PROXY_CONTAINER}:4000" + # --------------------------------------------------------------------------- # Exceptions @@ -295,6 +306,7 @@ class HarnessRunPlan: network_policy: NetworkPolicy proxy_host: str proxy_port: int + sandbox_network: str = SANDBOX_NETWORK @dataclass(frozen=True) @@ -357,28 +369,35 @@ def build_docker_run_kwargs(plan: HarnessRunPlan) -> dict[str, object]: if plan.network_policy is NetworkPolicy.NONE: kwargs["network_mode"] = "none" else: - # PROXY_ONLY — the security-sensitive bridge is an open RFC-006 decision. - raise NetworkPolicyNotConfigured( - "PROXY_ONLY egress bridge is an open RFC-006 decision " - "(sidecar-in-container-net vs host-gateway firewall). Wire it in " - f"Phase 2 for proxy {plan.proxy_host}:{plan.proxy_port}." - ) + # PROXY_ONLY — RFC-006 decision A: join the Docker `internal` sandbox + # network where the LiteLLM proxy is the only reachable host (bastion). + # No host route on that net, so this is the entire egress surface. + if not plan.sandbox_network: + raise NetworkPolicyNotConfigured( + "PROXY_ONLY requires a sandbox_network (RFC-006 decision A: the " + "Docker internal bastion network the proxy is attached to)" + ) + kwargs["network"] = plan.sandbox_network return kwargs class DockerHarnessLauncher: - """Real HarnessLauncher — RFC-006 Phase 2 (NOT wired in Phase 1). - - Present as the typed seam + provenance anchor; ``launch()`` raises until - Phase 2 builds the CLI sandbox images and resolves the proxy-only bridge. - The pure ``build_docker_run_kwargs`` above is already testable. + """Real HarnessLauncher — lands in RFC-006 Phase 3 (first real run). + + The image (``pollmevals-harness-aider``) and the network (decision A: the + Docker internal bastion net) are ready as of Phase 2. ``launch()`` itself — + docker-py run + git-diff patch capture + proxy-spend token metering — is + built and validated end-to-end against the first real ``aider x qwen x + be_01`` run, so it isn't written blind here. ``build_docker_run_kwargs`` + above (the security-sensitive surface) is already wired + tested. """ async def launch(self, plan: HarnessRunPlan) -> HarnessRunOutcome: raise NotImplementedError( - "DockerHarnessLauncher.launch is RFC-006 Phase 2 — needs the CLI " - "sandbox image + the proxy-only network bridge. Phase 1 uses " - "FakeHarnessLauncher." + "DockerHarnessLauncher.launch lands in RFC-006 Phase 3 (first real " + "run): docker-py run on the internal sandbox network + git-diff " + "patch capture + proxy-spend token metering, validated against " + "aider x qwen x be_01. Phase 1/2 use FakeHarnessLauncher." ) @@ -502,7 +521,7 @@ def __init__( self, *, launcher: HarnessLauncher, - proxy_base_url: str = DEFAULT_PROXY_BASE_URL, + proxy_base_url: str = SANDBOX_PROXY_BASE_URL, api_key: str = "", pricing_snapshot: dict[str, PricingTuple] | None = None, image_resolver: Callable[[str], str] = default_image_for_cli, diff --git a/apps/eval-core-py/tests/test_stack_executor.py b/apps/eval-core-py/tests/test_stack_executor.py index 6d2c038..53cb468 100644 --- a/apps/eval-core-py/tests/test_stack_executor.py +++ b/apps/eval-core-py/tests/test_stack_executor.py @@ -6,6 +6,7 @@ from __future__ import annotations +import dataclasses from datetime import UTC, datetime from decimal import Decimal from pathlib import Path @@ -15,6 +16,8 @@ from src.orchestrator.cost import PricingTuple from src.orchestrator.stack_executor import ( DEFAULT_PROXY_BASE_URL, + SANDBOX_NETWORK, + SANDBOX_PROXY_BASE_URL, DockerHarnessLauncher, ExecStatus, ExecutionMode, @@ -242,10 +245,17 @@ def test_security_flags_locked_down(self, tmp_path: Path) -> None: assert kw["security_opt"] == ["no-new-privileges:true"] assert kw["network_mode"] == "none" - def test_proxy_only_is_an_open_decision(self, tmp_path: Path) -> None: - # The crux: the proxy-only egress bridge is not baked in until Phase 2. - with pytest.raises(NetworkPolicyNotConfigured, match="open RFC-006 decision"): - build_docker_run_kwargs(_plan(NetworkPolicy.PROXY_ONLY, tmp_path)) + def test_proxy_only_joins_internal_bastion_network(self, tmp_path: Path) -> None: + # Decision A: PROXY_ONLY -> join the Docker `internal` sandbox network + # (the proxy is the only reachable host). No "none" network_mode. + kw = build_docker_run_kwargs(_plan(NetworkPolicy.PROXY_ONLY, tmp_path)) + assert kw["network"] == SANDBOX_NETWORK + assert "network_mode" not in kw + + def test_proxy_only_without_network_raises(self, tmp_path: Path) -> None: + plan = dataclasses.replace(_plan(NetworkPolicy.PROXY_ONLY, tmp_path), sandbox_network="") + with pytest.raises(NetworkPolicyNotConfigured, match="decision A"): + build_docker_run_kwargs(plan) # --------------------------------------------------------------------------- @@ -297,8 +307,12 @@ async def test_command_assembled_and_proxy_only(self, tmp_path: Path) -> None: ] assert plan.network_policy is NetworkPolicy.PROXY_ONLY assert plan.mount_dir == tmp_path - assert plan.proxy_host == "localhost" + assert plan.sandbox_network == SANDBOX_NETWORK + # Default executor proxy is the in-sandbox bastion (container name, not + # localhost — the internal net has no host route). + assert plan.proxy_host == "pollmevals-litellm-proxy" assert plan.proxy_port == 4000 + assert plan.environment["OPENAI_API_BASE"] == "http://pollmevals-litellm-proxy:4000/v1" @pytest.mark.asyncio async def test_timeout_is_min_of_request_and_limit(self, tmp_path: Path) -> None: @@ -372,14 +386,14 @@ async def launch(self, plan: HarnessRunPlan) -> HarnessRunOutcome: # --------------------------------------------------------------------------- -# Phase-2 seam: Docker launcher is intentionally not wired yet +# Phase-3 seam: the real Docker launcher lands with the first real run # --------------------------------------------------------------------------- -class TestPhase2Seam: +class TestPhase3Seam: @pytest.mark.asyncio async def test_docker_launcher_not_wired(self, tmp_path: Path) -> None: - with pytest.raises(NotImplementedError, match="Phase 2"): + with pytest.raises(NotImplementedError, match="Phase 3"): await DockerHarnessLauncher().launch(_plan(NetworkPolicy.NONE, tmp_path)) def test_default_image_name(self) -> None: @@ -387,3 +401,7 @@ def test_default_image_name(self) -> None: def test_default_proxy_base_url(self) -> None: assert DEFAULT_PROXY_BASE_URL == "http://localhost:4000" + + def test_sandbox_proxy_is_container_addressed(self) -> None: + # In-sandbox the proxy is the bastion container, not localhost. + assert SANDBOX_PROXY_BASE_URL == "http://pollmevals-litellm-proxy:4000" diff --git a/infra/docker-compose.litellm.yml b/infra/docker-compose.litellm.yml index d45ce3c..2817877 100644 --- a/infra/docker-compose.litellm.yml +++ b/infra/docker-compose.litellm.yml @@ -100,6 +100,14 @@ services: volumes: - ./litellm-config.yaml:/app/config.yaml:ro + # RFC-006 decision A (2026-06-02): the proxy is the bastion. It joins BOTH + # the default net (Postgres + OpenRouter egress) AND the internal sandbox + # net, where it is the only reachable host for harness containers. A harness + # on `pollmevals-sandbox` can reach the proxy but has NO external route. + networks: + - default + - sandbox + command: - "--config" - "/app/config.yaml" @@ -123,3 +131,10 @@ volumes: networks: default: name: pollmevals-dev + # RFC-006 decision A: `internal` = no external route. Harness sandboxes + # (RFC-006 Half A) join this net; the proxy is the only host attached to it, + # so a harness can reach ONLY the metered proxy — no un-metered egress, and + # no NET_ADMIN needed (cap_drop ALL holds). Portable Linux/macOS/CI. + sandbox: + name: pollmevals-sandbox + internal: true diff --git a/infra/docker/harness-aider/Dockerfile b/infra/docker/harness-aider/Dockerfile new file mode 100644 index 0000000..bd58bd1 --- /dev/null +++ b/infra/docker/harness-aider/Dockerfile @@ -0,0 +1,71 @@ +# syntax=docker/dockerfile:1.7 +# +# pollmevals-harness-aider -- sandboxed aider CLI image (RFC-006 Half A, candidate side). +# +# Built once, used by StackExecutor / DockerHarnessLauncher to run the aider +# harness against a task. Unlike the Half B evaluator image (eval-ts), the +# candidate side is the OPPOSITE shape: +# * /workspace is WRITABLE (the harness must edit files to produce a patch) +# * the container joins the `pollmevals-sandbox` INTERNAL network so it can +# reach ONLY the LiteLLM proxy (bastion) -- no general egress. The harness +# never gets the upstream provider key; it talks to the proxy, which meters +# every token (RFC-006 invariant: no un-metered egress). +# +# Pinned dependencies (Library-first; bump deliberately, never floating): +# python 3.12-slim (aider-chat requires <3.13,>=3.10) +# aider-chat 0.86.2 (latest on PyPI 2026-06-02) +# git (Debian stable) -- aider tracks edits / computes diffs via git +# +# Build: +# docker build -t pollmevals-harness-aider:0.1.0 infra/docker/harness-aider/ +# (or: make harness-image-aider) +# +# Run (referenced from DockerHarnessLauncher -- not invoked by humans directly): +# docker run --rm \ +# --network=pollmevals-sandbox \ +# --cap-drop=ALL --security-opt=no-new-privileges:true \ +# --memory=2g --pids-limit=256 \ +# -v $(pwd)/snapshot:/workspace \ +# -e OPENAI_API_BASE=http://pollmevals-litellm-proxy:4000/v1 \ +# -e OPENAI_API_KEY=$LITELLM_MASTER_KEY \ +# pollmevals-harness-aider:0.1.0 \ +# aider --no-pretty --yes --model openai/ --message "" +# +# Smoke (no proxy needed): +# docker run --rm pollmevals-harness-aider:0.1.0 # -> prints aider version + +FROM python:3.12-slim AS base + +# git: aider needs it to track edits and compute diffs. +# ca-certificates: TLS to the proxy bastion. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Pinned aider (library-first; bump deliberately). Installed at build time so +# the run-time container needs no PyPI access on the internal network. +RUN pip install --no-cache-dir aider-chat==0.86.2 + +# Non-root user: defense-in-depth on top of --cap-drop=ALL and +# --security-opt=no-new-privileges. uid 1000 matches the host bind owner so the +# produced patch is writable back on the host snapshot. +RUN useradd --create-home --uid 1000 harness +USER harness +WORKDIR /workspace + +# Default git identity so aider's edit-tracking never fails on a fresh snapshot. +RUN git config --global user.email "harness@pollmevals.local" \ + && git config --global user.name "pollmevals-harness" \ + && git config --global init.defaultBranch main \ + && git config --global --add safe.directory /workspace + +# aider also disables its own analytics / auto-update phone-home on the isolated +# network; set the env so it never blocks waiting on a network call it can't make. +ENV AIDER_ANALYTICS=false \ + AIDER_CHECK_UPDATE=false + +# Default command is a harmless version probe; DockerHarnessLauncher overrides +# `command` with the full aider invocation (and, in Phase 3, a git-diff capture +# wrapper). Keeping NO restrictive ENTRYPOINT lets the launcher pass either an +# argv list or an explicit `sh -c` wrapper. +CMD ["aider", "--version"] From 639d488bcc8e404946d0d78ce394bf3262b5cdd4 Mon Sep 17 00:00:00 2001 From: gogocat Date: Tue, 2 Jun 2026 20:45:01 +0300 Subject: [PATCH 3/7] =?UTF-8?q?feat(executor):=20Phase=203=20=E2=80=94=20D?= =?UTF-8?q?ockerHarnessLauncher=20+=20FIRST=20real=20patch=20(aider=C3=97q?= =?UTF-8?q?wen=C3=97be=5F01)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC-006 Phase 3: the candidate pipeline (Half A) runs end-to-end on real money. - DockerHarnessLauncher.launch: docker-py run on the pollmevals-sandbox bastion net (decision A) against a writable /workspace; host-side git base + then `git add -A && git diff --cached ` robustly captures the patch (committed + working-tree + new files) without shell-quoting the prompt. Auto-discovers the Docker Desktop socket (~/.docker/run) since docker-py's from_env() defaults to /var/run and misses it. - scripts/stack_exec_live_smoke.py: --plumbing ($0 no-model docker/patch check) + --confirm-spend (real run, gated like smoke_run.py). - Tests: git base/capture + token-parse plumbing (host git, no Docker); the container run itself is validated by the live smoke. FIRST REAL NUMBER — aider × qwen-3-14b × be_01 via the metered bastion proxy: status=ok, 208-line patch (a real Express JWT auth middleware), cost=$0.000626, in=1400/out=2200 tokens, ~55s. Bastion proven end-to-end: a sandbox container reaches the proxy (200) but NOT the internet (DNS fails) — no un-metered egress. Gates: ruff + mypy --strict clean; 39 executor tests; full suite 694 passed. Refs: rfc-006 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scripts/stack_exec_live_smoke.py | 179 ++++++++++++++++++ .../src/orchestrator/stack_executor.py | 165 ++++++++++++++-- .../eval-core-py/tests/test_stack_executor.py | 33 +++- 3 files changed, 357 insertions(+), 20 deletions(-) create mode 100644 apps/eval-core-py/scripts/stack_exec_live_smoke.py diff --git a/apps/eval-core-py/scripts/stack_exec_live_smoke.py b/apps/eval-core-py/scripts/stack_exec_live_smoke.py new file mode 100644 index 0000000..34b3833 --- /dev/null +++ b/apps/eval-core-py/scripts/stack_exec_live_smoke.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +"""Live smoke: aider x qwen-3-14b x be_01 -> a REAL patch via the bastion proxy. + +RFC-006 Phase 3 first real run (Half A: harness -> patch). Proves the whole +candidate pipeline end-to-end with real money: StackExecutor -> the aider image +on the `pollmevals-sandbox` internal net -> the LiteLLM proxy (metered) -> a +captured git diff. + +Cost: a fraction of a cent on qwen-3-14b. Spend is real, so it is gated behind +--confirm-spend (mirrors scripts/smoke_run.py). + +Prerequisites: + make stack-up # proxy healthy + make sandbox-net-up # proxy attached to pollmevals-sandbox (decision A) + make harness-image-aider + +Run: + uv run --project apps/eval-core-py python \ + apps/eval-core-py/scripts/stack_exec_live_smoke.py --confirm-spend +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import shutil +import sys +import tempfile +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path + +import yaml + +REPO = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(REPO / "apps" / "eval-core-py")) + +from src.orchestrator.cost import PricingTuple # noqa: E402 +from src.orchestrator.stack_executor import ( # noqa: E402 + SANDBOX_NETWORK, + DockerHarnessLauncher, + ExecStatus, + HarnessRunPlan, + NetworkPolicy, + StackAdapter, + StackExecRequest, + StackExecutor, +) + +# qwen-3-14b approximate OpenRouter pricing (per Mtoken). Informational for the +# smoke; the proxy is the metered source of truth for production reconciliation. +_QWEN_PRICING = PricingTuple( + model_id="openrouter/qwen/qwen-3-14b", + input_per_mtoken_usd=Decimal("0.07"), + output_per_mtoken_usd=Decimal("0.24"), + snapshot_at=datetime(2026, 6, 2, tzinfo=UTC), +) + + +def _load_env(repo: Path) -> None: + envf = repo / ".env" + if not envf.exists(): + return + for line in envf.read_text(encoding="utf-8").splitlines(): + s = line.strip() + if not s or s.startswith("#") or "=" not in s: + continue + k, _, v = s.partition("=") + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def _seed_candidate_snapshot(dst: Path, pack: Path) -> None: + """Seed a be_01 CANDIDATE workspace: pinned deps only — NO gold, NO tests.""" + gold = pack / "gold" + for f in ("package.json", "tsconfig.json"): + shutil.copy(gold / f, dst / f) + (dst / "solution.ts").write_text( + "// Implement the Express JWT auth middleware here (see the task prompt).\n", + encoding="utf-8", + ) + + +async def _plumbing_check() -> int: + """$0 end-to-end check of _run_sync: container on the bastion net writes to + /workspace via a NO-MODEL command; the launcher must capture the diff.""" + snap = Path(tempfile.mkdtemp(prefix="pollmevals-plumb-")) + (snap / "seed.txt").write_text("seed\n", encoding="utf-8") + plan = HarnessRunPlan( + image="pollmevals-harness-aider:0.1.0", + command=["sh", "-c", "echo 'export const x = 1;' > /workspace/new.ts"], + workdir="/workspace", + mount_dir=snap, + environment={}, + config_files={}, + timeout_s=60, + network_policy=NetworkPolicy.PROXY_ONLY, + proxy_host="pollmevals-litellm-proxy", + proxy_port=4000, + sandbox_network=SANDBOX_NETWORK, + ) + outcome = await DockerHarnessLauncher().launch(plan) + print(f"exit={outcome.exit_code} timed_out={outcome.timed_out} wall_ms={outcome.wall_ms}") + print("--- captured patch ---\n" + outcome.patch) + ok = "new.ts" in outcome.patch and outcome.exit_code == 0 and not outcome.timed_out + print("PLUMBING OK ✅" if ok else "PLUMBING FAILED ❌") + return 0 if ok else 1 + + +async def _main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--confirm-spend", action="store_true", help="actually run (real $)") + ap.add_argument("--plumbing", action="store_true", help="$0 no-model docker/patch check") + ap.add_argument("--model-alias", default="qwen-3-14b") + args = ap.parse_args() + + if args.plumbing: + return await _plumbing_check() + + _load_env(REPO) + master_key = os.environ.get("LITELLM_MASTER_KEY", "") + if not master_key: + print("ERROR: LITELLM_MASTER_KEY not set (.env)", file=sys.stderr) + return 2 + + if not args.confirm_spend: + print("DRY: pass --confirm-spend to run aider x qwen x be_01 for real (~<$0.01).") + return 0 + + pack = REPO / "evals" / "task-packs" / "be_01_jwt_auth" + prompt = str(yaml.safe_load((pack / "task.yaml").read_text())["prompt_template"]) + adapter = StackAdapter.from_yaml_path(REPO / "stacks" / "aider" / "stack.yaml") + + snapshot = Path(tempfile.mkdtemp(prefix="pollmevals-be01-")) + _seed_candidate_snapshot(snapshot, pack) + + executor = StackExecutor( + launcher=DockerHarnessLauncher(), + api_key=master_key, + pricing_snapshot={"openrouter/qwen/qwen-3-14b": _QWEN_PRICING}, + ) + request = StackExecRequest( + eval_id="smoke-aider-qwen-be01", + model_id="openrouter/qwen/qwen-3-14b", + model_alias=args.model_alias, + stack=adapter, + task_id="be_01_jwt_auth", + task_prompt=prompt, + repo_snapshot_dir=snapshot, + seed=1, + timeout_s=600, + ) + + print(f"Running aider x {args.model_alias} x be_01 in {snapshot} ...") + result = await executor.execute(request) + + print("\n=== RESULT ===") + print(f"status: {result.status}") + print(f"error_detail: {result.error_detail}") + print(f"tokens: in={result.input_tokens} out={result.output_tokens}") + print(f"cost_usd: {result.cost_usd}") + print(f"wall_ms: {result.wall_ms}") + if result.patch: + out = snapshot / "captured.patch" + out.write_text(result.patch, encoding="utf-8") + nlines = result.patch.count("\n") + print(f"patch: {nlines} lines -> {out}") + print("\n--- patch head (40 lines) ---") + print("\n".join(result.patch.splitlines()[:40])) + else: + print("patch: (none)") + print("\n--- trace tail (30 lines) ---") + print("\n".join(result.trace.splitlines()[-30:])) + + return 0 if result.status is ExecStatus.OK else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(_main())) diff --git a/apps/eval-core-py/src/orchestrator/stack_executor.py b/apps/eval-core-py/src/orchestrator/stack_executor.py index 5c9e7ac..f637498 100644 --- a/apps/eval-core-py/src/orchestrator/stack_executor.py +++ b/apps/eval-core-py/src/orchestrator/stack_executor.py @@ -34,14 +34,20 @@ from __future__ import annotations +import asyncio +import contextlib import logging +import os +import re +import subprocess +import time from collections.abc import Callable from dataclasses import dataclass, field from datetime import UTC, datetime from decimal import Decimal from enum import StrEnum from pathlib import Path -from typing import Protocol, runtime_checkable +from typing import Any, Protocol, runtime_checkable import yaml from pydantic import BaseModel, ConfigDict, Field @@ -381,25 +387,156 @@ def build_docker_run_kwargs(plan: HarnessRunPlan) -> dict[str, object]: return kwargs +# aider prints a usage line like "Tokens: 1.2k sent, 850 received." at the end +# of a run. Best-effort parse for the first run; proxy-spend reconciliation +# (the harness-agnostic source) is the Phase 4 refinement. +_AIDER_TOKENS_RE = re.compile( + r"Tokens:\s*([\d.]+)\s*([km]?)\s*sent.*?([\d.]+)\s*([km]?)\s*received", + re.IGNORECASE, +) + + +def _scale(value: str, suffix: str) -> int: + mult = {"k": 1_000, "m": 1_000_000}.get(suffix.lower(), 1) + return int(float(value) * mult) + + class DockerHarnessLauncher: - """Real HarnessLauncher — lands in RFC-006 Phase 3 (first real run). - - The image (``pollmevals-harness-aider``) and the network (decision A: the - Docker internal bastion net) are ready as of Phase 2. ``launch()`` itself — - docker-py run + git-diff patch capture + proxy-spend token metering — is - built and validated end-to-end against the first real ``aider x qwen x - be_01`` run, so it isn't written blind here. ``build_docker_run_kwargs`` - above (the security-sensitive surface) is already wired + tested. + """Real HarnessLauncher (RFC-006 Phase 3). + + Runs the harness in a fresh container on the internal bastion network + (decision A) against a writable /workspace bind, then captures the produced + patch HOST-side via git so the harness command stays a clean argv list (no + shell-quoting of the multi-line task prompt): + + 1. host-side: ensure /workspace is a git repo + record a base commit + 2. container: run ``plan.command`` (argv, no shell) — the harness edits + /workspace and may auto-commit + 3. host-side: ``git add -A && git diff --cached `` — robustly + captures committed + working-tree + new files relative to the base + + Token metering is best-effort from the harness self-report for the first + run; the proxy is the metered source of truth for cost reconciliation later. + + docker-py is synchronous; ``launch`` wraps the blocking run in + ``asyncio.to_thread`` (same discipline as SandboxRun in Half B). """ + def __init__(self) -> None: + self._client: Any = None + async def launch(self, plan: HarnessRunPlan) -> HarnessRunOutcome: - raise NotImplementedError( - "DockerHarnessLauncher.launch lands in RFC-006 Phase 3 (first real " - "run): docker-py run on the internal sandbox network + git-diff " - "patch capture + proxy-spend token metering, validated against " - "aider x qwen x be_01. Phase 1/2 use FakeHarnessLauncher." + return await asyncio.to_thread(self._run_sync, plan) + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + def _ensure_client(self) -> Any: + if self._client is None: + try: + import docker # type: ignore[import-untyped] + except ImportError as exc: + raise ImportError( + "docker SDK not installed; add `docker>=7.1,<8` to " + "apps/eval-core-py/pyproject.toml dependencies." + ) from exc + # docker-py's from_env() defaults to /var/run/docker.sock and does + # NOT read the docker CLI context. On macOS Docker Desktop the socket + # lives under ~/.docker/run, so discover it like the CLI does. + if "DOCKER_HOST" not in os.environ: + for cand in ( + Path.home() / ".docker" / "run" / "docker.sock", + Path("/var/run/docker.sock"), + ): + if cand.exists(): + os.environ["DOCKER_HOST"] = f"unix://{cand}" + break + self._client = docker.from_env() + return self._client + + @staticmethod + def _git(workspace: Path, *args: str) -> subprocess.CompletedProcess[str]: + # -c safe.directory=* avoids "dubious ownership" when the container + # (uid 1000) and the host user differ on the bind-mounted .git. + return subprocess.run( + ["git", "-c", "safe.directory=*", "-C", str(workspace), *args], + check=False, + capture_output=True, + text=True, + ) + + def _ensure_git_base(self, workspace: Path) -> str: + """Init (if needed) + commit the snapshot AS GIVEN, return base SHA.""" + if not (workspace / ".git").exists(): + self._git(workspace, "init", "-q") + self._git(workspace, "config", "user.email", "harness@pollmevals.local") + self._git(workspace, "config", "user.name", "pollmevals-harness") + self._git(workspace, "add", "-A") + self._git(workspace, "commit", "-q", "-m", "pollmevals-base", "--allow-empty") + return self._git(workspace, "rev-parse", "HEAD").stdout.strip() + + def _capture_patch(self, workspace: Path, base_sha: str) -> str: + """Unified diff of everything (committed + working + new) vs base.""" + self._git(workspace, "add", "-A") + return self._git(workspace, "diff", "--cached", base_sha).stdout + + def _run_sync(self, plan: HarnessRunPlan) -> HarnessRunOutcome: + workspace = plan.mount_dir + base_sha = self._ensure_git_base(workspace) + + client = self._ensure_client() + run_kwargs = build_docker_run_kwargs(plan) + run_kwargs["command"] = plan.command # argv list, no shell + + start_ns = time.monotonic_ns() + container = client.containers.run(**run_kwargs) + + timed_out = False + try: + wait_result = container.wait(timeout=plan.timeout_s) + exit_code = int(wait_result.get("StatusCode", -1)) + except Exception as exc: # docker-py ReadTimeout on overrun + logger.warning("Harness container timed out after %ds: %s", plan.timeout_s, exc) + timed_out = True + with contextlib.suppress(Exception): + container.kill() + exit_code = 137 + + try: + stdout = container.logs(stdout=True, stderr=False).decode(errors="replace") + stderr = container.logs(stdout=False, stderr=True).decode(errors="replace") + except Exception as exc: + logger.warning("Harness log capture failed: %s", exc) + stdout, stderr = "", "" + + with contextlib.suppress(Exception): + container.remove(v=True, force=True) + + wall_ms = (time.monotonic_ns() - start_ns) // 1_000_000 + patch = self._capture_patch(workspace, base_sha) + in_tok, out_tok = self._parse_tokens(stdout + "\n" + stderr) + + return HarnessRunOutcome( + exit_code=exit_code, + patch=patch, + trace=stdout, + stderr=stderr, + input_tokens=in_tok, + output_tokens=out_tok, + tool_calls=0, + wall_ms=int(wall_ms), + timed_out=timed_out, ) + @staticmethod + def _parse_tokens(text: str) -> tuple[int, int]: + """Best-effort token counts from the harness self-report (else 0/0).""" + m = _AIDER_TOKENS_RE.search(text) + if m is None: + return 0, 0 + return _scale(m.group(1), m.group(2)), _scale(m.group(3), m.group(4)) + @dataclass class FakeHarnessLauncher: diff --git a/apps/eval-core-py/tests/test_stack_executor.py b/apps/eval-core-py/tests/test_stack_executor.py index 53cb468..6a6dab8 100644 --- a/apps/eval-core-py/tests/test_stack_executor.py +++ b/apps/eval-core-py/tests/test_stack_executor.py @@ -386,15 +386,36 @@ async def launch(self, plan: HarnessRunPlan) -> HarnessRunOutcome: # --------------------------------------------------------------------------- -# Phase-3 seam: the real Docker launcher lands with the first real run +# DockerHarnessLauncher — the git patch-capture + token parse plumbing is +# testable WITHOUT Docker (host git + pure parsing). The container run itself +# is validated by the live smoke (scripts/stack_exec_live_smoke.py). # --------------------------------------------------------------------------- -class TestPhase3Seam: - @pytest.mark.asyncio - async def test_docker_launcher_not_wired(self, tmp_path: Path) -> None: - with pytest.raises(NotImplementedError, match="Phase 3"): - await DockerHarnessLauncher().launch(_plan(NetworkPolicy.NONE, tmp_path)) +class TestDockerLauncherPlumbing: + def test_git_base_then_capture_new_file(self, tmp_path: Path) -> None: + (tmp_path / "existing.txt").write_text("v1\n") + launcher = DockerHarnessLauncher() + base = launcher._ensure_git_base(tmp_path) + assert len(base) == 40 # a full git sha + # simulate the harness editing the tree: modify + add a NEW file + (tmp_path / "existing.txt").write_text("v2\n") + (tmp_path / "solution.ts").write_text("export const x = 1;\n") + patch = launcher._capture_patch(tmp_path, base) + assert "solution.ts" in patch # new file captured + assert "+export const x = 1;" in patch + assert "-v1" in patch and "+v2" in patch # modification captured + + def test_capture_empty_when_no_changes(self, tmp_path: Path) -> None: + (tmp_path / "a.txt").write_text("same\n") + launcher = DockerHarnessLauncher() + base = launcher._ensure_git_base(tmp_path) + assert launcher._capture_patch(tmp_path, base).strip() == "" + + def test_parse_aider_tokens(self) -> None: + launcher = DockerHarnessLauncher() + assert launcher._parse_tokens("Tokens: 1.2k sent, 850 received.") == (1200, 850) + assert launcher._parse_tokens("no usage line here") == (0, 0) def test_default_image_name(self) -> None: assert default_image_for_cli("aider") == "pollmevals-harness-aider:0.1.0" From ae5b48b14502d89a25b4cdb88375a36ab37c970e Mon Sep 17 00:00:00 2001 From: gogocat Date: Tue, 2 Jun 2026 21:29:06 +0300 Subject: [PATCH 4/7] =?UTF-8?q?feat(executor):=20Phase=204a=20=E2=80=94=20?= =?UTF-8?q?Half=20A=E2=86=92B=20scoring=20bridge=20+=20FIRST=20scored=20nu?= =?UTF-8?q?mber?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC-006 Phase 4. Bridges StackExecResult (Half A patch) → EvalResult so the judge panel (and evaluators) score a (model × harness × task) run. - src/orchestrator/stack_scoring.py: changed_files (parse diff, filter harness noise .aider/.gitignore), extract_submission (final content of changed source files — code, not diff), exec_result_to_eval_result (write submission as the raw_output artifact + map to EvalRow; rejects non-OK runs). GridRunner dispatch-by-stack will reuse this same path. - scripts/stack_score_live_smoke.py: executor → bridge → judge panel → scored number. Anchors cwd at repo root (the panel resolves rubric.yaml via cwd). - 6 bridge unit tests (no Docker/judges). FIRST SCORED NUMBER — aider × qwen-3-14b × be_01, judged (inversion-free, since the be_01 deterministic evaluators invert per EVID-027): claude-sonnet 5.27 · gemini-3-flash 7.33 · gpt-5-mini 0.00* · cost $0.070 (*DEFECT: gpt-5-mini truncated its rubric JSON at the 2048 cap → parse-fail → 0.0 fallback, dragging the median + alpha. Trustworthy 2-judge signal ≈ 6.3. Fix is reasoning_effort per EVID-023, NOT raising the cap — separate judge work.) Gates: ruff + mypy --strict clean; 6 bridge tests; full suite 700 passed. Refs: rfc-006 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../scripts/stack_score_live_smoke.py | 164 ++++++++++++++++++ .../src/orchestrator/stack_scoring.py | 160 +++++++++++++++++ apps/eval-core-py/tests/test_stack_scoring.py | 132 ++++++++++++++ 3 files changed, 456 insertions(+) create mode 100644 apps/eval-core-py/scripts/stack_score_live_smoke.py create mode 100644 apps/eval-core-py/src/orchestrator/stack_scoring.py create mode 100644 apps/eval-core-py/tests/test_stack_scoring.py diff --git a/apps/eval-core-py/scripts/stack_score_live_smoke.py b/apps/eval-core-py/scripts/stack_score_live_smoke.py new file mode 100644 index 0000000..cffc8d2 --- /dev/null +++ b/apps/eval-core-py/scripts/stack_score_live_smoke.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +"""Live: aider x qwen-3-14b x be_01 -> patch -> JUDGE PANEL -> FIRST scored number. + +RFC-006 Phase 4. Chains Half A (StackExecutor) + the Half A->B bridge +(stack_scoring) + the operational judge panel (#28). Judges are used (not the +be_01 deterministic evaluators, which invert — EVID-027); judged-subjective +scoring is also POLLMEVALS' edge. + +Cost: ~$0.0006 (aider on qwen) + ~$0.05 (3 judges). Gated by --confirm-spend. + +Prereqs: make stack-up && make sandbox-net-up && make harness-image-aider. +Run: + uv run --project apps/eval-core-py python \ + apps/eval-core-py/scripts/stack_score_live_smoke.py --confirm-spend +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import shutil +import statistics +import sys +import tempfile +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path + +import yaml + +REPO = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(REPO / "apps" / "eval-core-py")) + +from src.orchestrator.cost import PricingTuple # noqa: E402 +from src.orchestrator.judge_panel import JudgePanel # noqa: E402 +from src.orchestrator.stack_executor import ( # noqa: E402 + DockerHarnessLauncher, + ExecStatus, + StackAdapter, + StackExecRequest, + StackExecutor, +) +from src.orchestrator.stack_scoring import ( # noqa: E402 + cost_with_judges, + exec_result_to_eval_result, +) + +_CANDIDATE = "openrouter/qwen/qwen-3-14b" +_TASK = "be_01_jwt_auth" +_JUDGES = ["claude-sonnet-4-6-judge", "gpt-5-mini-judge", "gemini-3-flash"] +_QWEN_PRICING = PricingTuple( + model_id=_CANDIDATE, + input_per_mtoken_usd=Decimal("0.07"), + output_per_mtoken_usd=Decimal("0.24"), + snapshot_at=datetime(2026, 6, 2, tzinfo=UTC), +) + + +def _load_env(repo: Path) -> None: + envf = repo / ".env" + if not envf.exists(): + return + for line in envf.read_text(encoding="utf-8").splitlines(): + s = line.strip() + if s and not s.startswith("#") and "=" in s: + k, _, v = s.partition("=") + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def _seed_candidate_snapshot(dst: Path, pack: Path) -> None: + """be_01 CANDIDATE workspace: pinned deps only — NO gold, NO tests.""" + for f in ("package.json", "tsconfig.json"): + shutil.copy(pack / "gold" / f, dst / f) + (dst / "solution.ts").write_text( + "// Implement the Express JWT auth middleware here (see the task prompt).\n", + encoding="utf-8", + ) + + +async def _main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--confirm-spend", action="store_true", help="real $ (aider + judges)") + args = ap.parse_args() + + # JudgePanel resolves rubric.yaml relative to cwd; anchor at the repo root. + os.chdir(REPO) + _load_env(REPO) + master_key = os.environ.get("LITELLM_MASTER_KEY", "") + if not master_key: + print("ERROR: LITELLM_MASTER_KEY not set (.env)", file=sys.stderr) + return 2 + if not args.confirm_spend: + print("DRY: pass --confirm-spend to run aider + 3 judges for real (~$0.05).") + return 0 + + pack = REPO / "evals" / "task-packs" / _TASK + prompt = str(yaml.safe_load((pack / "task.yaml").read_text())["prompt_template"]) + adapter = StackAdapter.from_yaml_path(REPO / "stacks" / "aider" / "stack.yaml") + + snapshot = Path(tempfile.mkdtemp(prefix="pollmevals-score-")) + _seed_candidate_snapshot(snapshot, pack) + artifacts = snapshot / "artifacts" + + # --- Half A: harness -> patch --- + print(f"[1/3] Half A: aider x qwen x {_TASK} ...") + executor = StackExecutor( + launcher=DockerHarnessLauncher(), + api_key=master_key, + pricing_snapshot={_CANDIDATE: _QWEN_PRICING}, + ) + request = StackExecRequest( + eval_id="score-aider-qwen-be01", + model_id=_CANDIDATE, + model_alias="qwen-3-14b", + stack=adapter, + task_id=_TASK, + task_prompt=prompt, + repo_snapshot_dir=snapshot, + seed=1, + timeout_s=600, + ) + exec_result = await executor.execute(request) + print( + f" status={exec_result.status} cost=${exec_result.cost_usd} " + f"wall={exec_result.wall_ms}ms" + ) + if exec_result.status is not ExecStatus.OK: + print(f" executor did not produce a patch: {exec_result.error_detail}") + return 1 + + # --- Bridge: Half A -> Half B --- + print("[2/3] bridge: StackExecResult -> EvalResult (write submission artifact)") + eval_result = exec_result_to_eval_result(exec_result, log_dir=artifacts) + + # --- Half B: judge panel (inversion-free) --- + print(f"[3/3] Half B: {len(_JUDGES)} judges score the patch via the be_01 rubric ...") + panel = JudgePanel( + judge_models=_JUDGES, + candidate_model_id=_CANDIDATE, + rubric_version="1.0", + ) + judgments = await panel.score(eval_result, _TASK) + agg = panel.aggregate(judgments) + + judge_cost = sum((j.cost_usd for j in judgments), Decimal("0")) + total = cost_with_judges(exec_result.cost_usd, judge_cost) + + print("\n=== FIRST SCORED (model x harness x task) NUMBER ===") + print(f"stack: aider x qwen-3-14b task: {_TASK}") + for j in judgments: + print(f" judge {j.judge_model_id:<24} total={j.total_score:5.2f} cost=${j.cost_usd}") + panel_median = statistics.median([j.total_score for j in judgments]) if judgments else 0.0 + print(f"panel median: {panel_median:.2f} / 10 (per-criterion: {agg.median_per_criterion})") + print( + f"alpha: point={agg.alpha_point} ci_lower={agg.alpha_ci_lower} " + f"status={agg.judge_status}" + ) + print(f"cost: harness=${exec_result.cost_usd} + judges=${judge_cost} = ${total}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(_main())) diff --git a/apps/eval-core-py/src/orchestrator/stack_scoring.py b/apps/eval-core-py/src/orchestrator/stack_scoring.py new file mode 100644 index 0000000..2f7aff3 --- /dev/null +++ b/apps/eval-core-py/src/orchestrator/stack_scoring.py @@ -0,0 +1,160 @@ +"""Bridge Half A (StackExecResult patch) -> Half B (judge panel / evaluators). + +RFC-006 Phase 4. Maps a ``StackExecResult`` (harness -> patch, Half A) into the +``EvalResult`` shape that ``JudgePanel.score`` (and the evaluators) consume, so a +``(model x harness x task)`` run yields a real score. GridRunner dispatch-by-stack +reuses this so CLI stacks flow through the SAME scoring path as ``raw-llm``. + +The judge panel reads the candidate's output from the ``raw_output`` artifact +URI (a file on disk), so the bridge writes the submission there. + +Submission = the FINAL content of the candidate's changed source files (not the +raw diff), with harness bookkeeping (``.aider*``, ``.gitignore``, +``.pollmevals*``) filtered out — code-quality judging wants the code, not a diff. +""" + +from __future__ import annotations + +import hashlib +import re +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path + +from src.contracts import ( + ArtifactRef, + EvalArtifactRefs, + EvalRow, + EvalStats, + EvalStatus, +) +from src.orchestrator.eval_caller import EvalRequest, EvalResult, compute_eval_id +from src.orchestrator.stack_executor import ExecStatus, StackExecResult + +# Harness bookkeeping files that are NOT part of the candidate submission. +_NOISE_RE = re.compile(r"(^|/)(\.aider|\.gitignore|\.pollmevals|node_modules/)") + +# Synthetic run_hash fragment for standalone (non-grid) scoring. GridRunner +# passes the real run_hash; this default keeps single-eval scoring deterministic. +_STANDALONE_RUN_HASH = "sha256:" + "5" * 64 + + +def changed_files(patch: str) -> list[str]: + """Return the candidate's changed source paths from a unified diff. + + Reads ``+++ b/`` headers; drops ``/dev/null`` (pure deletions) and + harness bookkeeping noise. + """ + out: list[str] = [] + for line in patch.splitlines(): + if line.startswith("+++ b/"): + path = line[len("+++ b/") :].strip() + if path and path != "/dev/null" and not _NOISE_RE.search(path): + out.append(path) + return out + + +def extract_submission(snapshot_dir: Path, patch: str) -> str: + """Concatenated final content of the candidate's changed source files. + + Each file is prefixed with a ``// === ===`` banner so a multi-file + submission stays legible to the judge. Files referenced by the patch but + missing on disk (e.g. deletions) are skipped. + """ + parts: list[str] = [] + for rel in changed_files(patch): + f = snapshot_dir / rel + if f.exists(): + body = f.read_text(encoding="utf-8", errors="replace") + parts.append(f"// === {rel} ===\n{body}") + return "\n\n".join(parts) + + +def _artifact(log_dir: Path, eval_id: str, label: str, content: str) -> ArtifactRef: + """Write *content* to a content-addressed file and return its ArtifactRef.""" + sha256 = hashlib.sha256(content.encode()).hexdigest() + mime = "application/json" if label == "evaluator_json" else "text/plain" + ext = ".json" if label == "evaluator_json" else ".txt" + dest = log_dir / eval_id / f"{label}-{sha256}{ext}" + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(content, encoding="utf-8") + return ArtifactRef( + sha256=sha256, + size_bytes=len(content.encode()), + uri=f"file://{dest}", + mime_type=mime, + ) + + +def exec_result_to_eval_result( + exec_result: StackExecResult, + *, + log_dir: Path, + run_hash: str = _STANDALONE_RUN_HASH, +) -> EvalResult: + """Map a StackExecResult into a judge-ready EvalResult. + + The candidate submission (changed source files) is written as the + ``raw_output`` + ``normalized_output`` artifacts so ``JudgePanel.score`` can + read it. ``stack_id`` is the adapter slug; cost/tokens carry over from Half A. + + Raises: + ValueError: the executor did not produce a patch (status != OK) — there + is nothing to score. + """ + req = exec_result.request + if exec_result.status is not ExecStatus.OK or not exec_result.patch: + raise ValueError( + f"cannot score a non-OK execution (status={exec_result.status}, eval_id={req.eval_id})" + ) + + stack_id = req.stack.slug + eval_id = compute_eval_id(run_hash, req.model_id, stack_id, req.task_id, req.seed) + submission = extract_submission(req.repo_snapshot_dir, exec_result.patch) + evaluator_json = ( + f'{{"eval_id":"{eval_id}","harness":"{req.stack.agent_cli}",' + f'"patch_bytes":{len(exec_result.patch.encode())}}}' + ) + + artifact_refs = EvalArtifactRefs( + raw_output=_artifact(log_dir, eval_id, "raw_output", submission), + normalized_output=_artifact(log_dir, eval_id, "normalized_output", submission.strip()), + evaluator_json=_artifact(log_dir, eval_id, "evaluator_json", evaluator_json), + ) + stats = EvalStats( + input_tokens=exec_result.input_tokens, + output_tokens=exec_result.output_tokens, + wall_clock_ms=exec_result.wall_ms, + cost_usd=exec_result.cost_usd, + ) + row = EvalRow( + eval_id=eval_id, + model_id=req.model_id, + stack_id=stack_id, + task_id=req.task_id, + seed=req.seed, + status=EvalStatus.SCORED, + artifact_refs=artifact_refs, + stats=stats, + started_at=exec_result.started_at, + completed_at=exec_result.completed_at, + ) + request = EvalRequest( + eval_id=eval_id, + model_id=req.model_id, + stack_id=stack_id, + task_id=req.task_id, + seed=req.seed, + ) + return EvalResult( + request=request, + eval_row=row, + exception=None, + started_at=exec_result.started_at, + completed_at=datetime.now(UTC), + ) + + +def cost_with_judges(exec_cost: Decimal, judgments_cost: Decimal) -> Decimal: + """Total eval cost = harness spend + judge spend (CONTEXT.md cost rule).""" + return exec_cost + judgments_cost diff --git a/apps/eval-core-py/tests/test_stack_scoring.py b/apps/eval-core-py/tests/test_stack_scoring.py new file mode 100644 index 0000000..39279f4 --- /dev/null +++ b/apps/eval-core-py/tests/test_stack_scoring.py @@ -0,0 +1,132 @@ +"""Unit tests for the Half A -> Half B scoring bridge (RFC-006 Phase 4). + +Pure mapping logic — no Docker, no judges, no network. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path + +import pytest + +from src.contracts import EvalStatus +from src.orchestrator.stack_executor import ( + ExecStatus, + StackAdapter, + StackExecRequest, + StackExecResult, +) +from src.orchestrator.stack_scoring import ( + changed_files, + exec_result_to_eval_result, + extract_submission, +) + +_ADAPTER_YAML = """ +slug: aider +agent_cli: aider +execution: + mode: repository_patch + command: aider +""" + +_PATCH = """diff --git a/.gitignore b/.gitignore +new file mode 100644 +--- /dev/null ++++ b/.gitignore +@@ -0,0 +1 @@ ++.aider* +diff --git a/solution.ts b/solution.ts +--- a/solution.ts ++++ b/solution.ts +@@ -1 +1,2 @@ +-// stub ++export const ok = true; +diff --git a/old.ts b/old.ts +--- a/old.ts ++++ /dev/null +""" + + +def _exec_result( + tmp_path: Path, *, status: ExecStatus = ExecStatus.OK, patch: str = _PATCH +) -> StackExecResult: + adapter = StackAdapter.from_yaml_text(_ADAPTER_YAML) + req = StackExecRequest( + eval_id="raw-id", + model_id="openrouter/qwen/qwen-3-14b", + model_alias="qwen-3-14b", + stack=adapter, + task_id="be_01_jwt_auth", + task_prompt="impl", + repo_snapshot_dir=tmp_path, + seed=1, + ) + return StackExecResult( + request=req, + status=status, + patch=patch, + trace="t", + cost_usd=Decimal("0.000626"), + input_tokens=1400, + output_tokens=2200, + tool_calls=0, + wall_ms=55000, + error_detail=None, + started_at=datetime.now(UTC), + completed_at=datetime.now(UTC), + ) + + +class TestChangedFiles: + def test_parses_and_filters_noise(self) -> None: + files = changed_files(_PATCH) + assert "solution.ts" in files + assert ".gitignore" not in files # harness bookkeeping filtered + assert "old.ts" not in files # deletion (+++ /dev/null) dropped + + def test_empty_patch(self) -> None: + assert changed_files("") == [] + + +class TestExtractSubmission: + def test_reads_changed_file_content(self, tmp_path: Path) -> None: + (tmp_path / "solution.ts").write_text("export const ok = true;\n") + sub = extract_submission(tmp_path, _PATCH) + assert "=== solution.ts ===" in sub + assert "export const ok = true;" in sub + + def test_skips_missing_files(self, tmp_path: Path) -> None: + # solution.ts referenced by the patch but not on disk -> skipped, no raise + assert extract_submission(tmp_path, _PATCH) == "" + + +class TestExecResultToEvalResult: + def test_maps_ok_result(self, tmp_path: Path) -> None: + (tmp_path / "solution.ts").write_text("export const ok = true;\n") + log_dir = tmp_path / "artifacts" + result = _exec_result(tmp_path) + ev = exec_result_to_eval_result(result, log_dir=log_dir) + + assert ev.eval_row is not None + row = ev.eval_row + assert row.stack_id == "aider" # adapter slug, not the raw eval_id stack + assert row.model_id == "openrouter/qwen/qwen-3-14b" + assert row.task_id == "be_01_jwt_auth" + assert row.status is EvalStatus.SCORED + assert len(row.eval_id) == 16 # 16-hex contract + # cost + tokens carried over from Half A + assert row.stats.cost_usd == Decimal("0.000626") + assert row.stats.input_tokens == 1400 + # the raw_output artifact file was written with the submission + uri = row.artifact_refs.raw_output.uri + assert uri.startswith("file://") + written = Path(uri[len("file://") :]).read_text() + assert "export const ok = true;" in written + + def test_rejects_non_ok(self, tmp_path: Path) -> None: + result = _exec_result(tmp_path, status=ExecStatus.NO_PATCH, patch="") + with pytest.raises(ValueError, match="non-OK execution"): + exec_result_to_eval_result(result, log_dir=tmp_path) From 9b8824dbb25dfd934b0def6d9feb8e272637b560 Mon Sep 17 00:00:00 2001 From: gogocat Date: Tue, 2 Jun 2026 21:30:25 +0300 Subject: [PATCH 5/7] =?UTF-8?q?docs(runbook):=2014=20=E2=80=94=20stack=20e?= =?UTF-8?q?xecutor=20(Half=20A)=20runbook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit How to run a real harness×model×task eval: the bastion network model (decision A), one-time setup (stack-up / sandbox-net-up / harness-image-aider), the $0 plumbing check + the two live smokes (Half A only, full chain to a scored number), how host-side git patch capture works, the be_01 judge-vs-evaluator caveat (EVID-027 inversion), known defects (gpt-5-mini truncation, proxy cost reconciliation, GridRunner dispatch), and the first real numbers. Refs: rfc-006 Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/04-runbook/14-stack-executor.md | 171 +++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 docs/04-runbook/14-stack-executor.md diff --git a/docs/04-runbook/14-stack-executor.md b/docs/04-runbook/14-stack-executor.md new file mode 100644 index 0000000..dc22b0f --- /dev/null +++ b/docs/04-runbook/14-stack-executor.md @@ -0,0 +1,171 @@ +# 14 — Stack Executor (Half A: harness → patch) + +How to run a **real** `harness × model × task` evaluation: drive an agent-CLI +harness (aider, …) against a task inside an isolated sandbox, capture the +produced **patch + trace + metered cost**, then score it. + +Built per **RFC-006**. Complements the existing pieces: + +- `05-stack-adapter-guide.md` — the `stack.yaml` adapter this reads. +- `09-sandbox-security.md` — the frozen sandbox policy (Half B side). +- `07-judge-panel.md` + `08-scoring-contract.md` — how the patch is scored. + +--- + +## The two-half sandbox + +``` +Half A — candidate (this doc): harness CLI ──► patch + StackExecutor + DockerHarnessLauncher (src/orchestrator/stack_executor.py) + +Half B — evaluator: produced code ──► scores + evaluators/ + evaluators/sandbox/runner.py (+ judge panel) + +Bridge: StackExecResult ──► EvalResult (src/orchestrator/stack_scoring.py) +``` + +`raw-llm` (L0, `execution.mode: direct_completion`) stays on `InspectEvalCaller`. +Every `repository_patch` stack (aider, claude-code, codex, …) runs through the +**StackExecutor**. + +--- + +## Network model — the bastion (RFC-006 decision A) + +The harness must reach the LiteLLM proxy (to pick the model + meter every token) +but must NOT reach anything else (no un-metered egress, no data exfil). + +Solution: a Docker **`internal`** network `pollmevals-sandbox` (no external +route). The proxy is attached to it as the **only reachable host** (a bastion); +it also stays on the default net to reach OpenRouter. The harness joins ONLY +`pollmevals-sandbox`. + +``` +[ harness container ] ──(pollmevals-sandbox, internal)──► [ litellm-proxy ] ──(default)──► OpenRouter + │ + └── no route to anything else (DNS for the open internet fails) +``` + +Consequences: +- No `NET_ADMIN` / iptables needed → `cap_drop ALL` holds. +- Portable Linux / macOS / CI (pure Docker topology, not firewall rules). +- In-sandbox the proxy is addressed by **container name** (`http://pollmevals-litellm-proxy:4000`), + NOT `localhost` — the internal net has no host route. + +--- + +## One-time setup + +```bash +make stack-up # Postgres + NATS + LiteLLM proxy (healthy) +make sandbox-net-up # create pollmevals-sandbox + attach the proxy (idempotent; + # for an already-running stack — no recreate needed) +make harness-image-aider # build pollmevals-harness-aider:0.1.0 (python+aider+git) +``` + +`make stack-up` also creates `pollmevals-sandbox` declaratively (it is in +`infra/docker-compose.litellm.yml`); `sandbox-net-up` is the imperative path for +a stack that is already up. + +Verify the bastion (no spend): + +```bash +# reaches the proxy (expect 200) +docker run --rm --network pollmevals-sandbox pollmevals-harness-aider:0.1.0 \ + python -c "import urllib.request; print(urllib.request.urlopen('http://pollmevals-litellm-proxy:4000/health/liveliness',timeout=8).status)" +# cannot reach the internet (expect DNS failure) +docker run --rm --network pollmevals-sandbox pollmevals-harness-aider:0.1.0 \ + python -c "import urllib.request; urllib.request.urlopen('https://example.com',timeout=6)" +``` + +--- + +## Running it + +All scripts are spend-gated (`--confirm-spend`) and require `LITELLM_MASTER_KEY` +in `.env` (the proxy key — NOT the upstream provider key). + +### Plumbing check (\$0 — no model call) + +```bash +uv run --project apps/eval-core-py python \ + apps/eval-core-py/scripts/stack_exec_live_smoke.py --plumbing +``` + +Runs a no-model command in the sandbox and asserts the launcher captures the +git diff. Use this to debug Docker / mount / network issues before spending. + +### Half A only — harness → patch (≈ \$0.001) + +```bash +uv run --project apps/eval-core-py python \ + apps/eval-core-py/scripts/stack_exec_live_smoke.py --confirm-spend +``` + +### Full chain — harness → patch → judge → scored number (≈ \$0.07) + +```bash +uv run --project apps/eval-core-py python \ + apps/eval-core-py/scripts/stack_score_live_smoke.py --confirm-spend +``` + +Run from the **repo root** (the judge panel resolves `evals/task-packs//rubric.yaml` +via cwd; the score script also `chdir`s to the repo root defensively). + +--- + +## How patch capture works + +The harness command is run as a **clean argv list** (no shell), so the +multi-line task prompt needs no quoting. The patch is captured **host-side** with +git, robust to whether the harness auto-commits: + +1. before the run: `git init` (if needed) + commit the snapshot AS GIVEN → base SHA +2. run the harness container (writable `/workspace` bind on the bastion net) +3. after: `git add -A && git diff --cached ` → the unified patch + (committed + working-tree + new files), with harness bookkeeping + (`.aider*`, `.gitignore`) filtered out by the scoring bridge + +Token/cost: best-effort from the harness self-report for now; the proxy is the +metered source of truth for reconciliation (a Phase-4 follow-up). + +--- + +## Scoring path (be_01 caveat) + +The bridge (`stack_scoring.exec_result_to_eval_result`) writes the candidate's +changed source files as the `raw_output` artifact, then the judge panel scores +it against the task rubric. + +**Use the judge panel for be_01, not the deterministic evaluators** — the be_01 +deterministic evaluators score-INVERT (broken > perfect, EVID-027). Judged +subjective scoring is also POLLMEVALS' edge. `doc_01` is judge-only by design; +`fe_01` has working dynamic evaluators (post-#24). + +--- + +## Known defects / follow-ups + +- **gpt-5-mini judge truncates** its rubric JSON at the 2048 `max_tokens` cap on + a 7-criterion coding rubric → parse-fail → 0.0 fallback, which drags the panel + median + Krippendorff α. Fix is `reasoning_effort` (cap reasoning tokens), NOT + raising the cap (EVID-023: the cap is bounded by the OpenRouter HTTP-402 + pre-reservation hazard). Until then, read the per-judge scores, not just the + median. +- **Proxy-metered cost reconciliation** — replace the best-effort harness + self-report with a proxy `/spend` query (harness-agnostic). +- **GridRunner dispatch-by-stack** — route CLI stacks to StackExecutor inside + GridRunner so a full grid run emits real Board data (RFC-006 Phase 4b). +- **More harnesses** — codex / claude-code / goose / openhands recipes are + registered but pending their per-stack smoke (`_PENDING_RECIPES`, Phase 5). + +--- + +## First real numbers (2026-06-02) + +| stack | task | result | cost | +|---|---|---|---| +| aider × qwen-3-14b | be_01 | patch: 208-line Express JWT middleware, `status=ok` | \$0.00063 | +| aider × qwen-3-14b | be_01 | judged: claude 5.27 · gemini 7.33 · gpt-5-mini 0.00\* | \$0.070 | + +\* gpt-5-mini truncation defect (above); trustworthy 2-judge signal ≈ 6.3. From 1c37f46917c39d239ecb9dd19e46a2d3cf0c82f9 Mon Sep 17 00:00:00 2001 From: gogocat Date: Tue, 2 Jun 2026 21:32:08 +0300 Subject: [PATCH 6/7] =?UTF-8?q?chore(forgeplan):=20EVID-049=20=E2=80=94=20?= =?UTF-8?q?StackExecutor=20live=20evidence=20(PASS),=20informs=20RFC-006?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First-real-run evidence pack: bastion proven ($0), Half A first patch (aider×qwen×be_01, $0.000626), first judged score (median ~6.3 on 2 trustworthy judges), and the gpt-5-mini truncation defect. verdict=PASS, CL3, evidence_type=live_integration_run. Activated; R_eff recomputed. Refs: evid-rfc-006-stackexecutor-live-first-real-harness-model-patch-judged-score Co-Authored-By: Claude Opus 4.8 (1M context) --- ...del-patch-judged-score-aider-qwen-be-01.md | 62 +++++++++++++++++++ ...candidate-side-patch-trace-metered-cost.md | 1 + 2 files changed, 63 insertions(+) create mode 100644 .forgeplan/evidence/EVID-049-rfc-006-stackexecutor-live-first-real-harness-model-patch-judged-score-aider-qwen-be-01.md diff --git a/.forgeplan/evidence/EVID-049-rfc-006-stackexecutor-live-first-real-harness-model-patch-judged-score-aider-qwen-be-01.md b/.forgeplan/evidence/EVID-049-rfc-006-stackexecutor-live-first-real-harness-model-patch-judged-score-aider-qwen-be-01.md new file mode 100644 index 0000000..ee2c828 --- /dev/null +++ b/.forgeplan/evidence/EVID-049-rfc-006-stackexecutor-live-first-real-harness-model-patch-judged-score-aider-qwen-be-01.md @@ -0,0 +1,62 @@ +--- +depth: standard +id: EVID-049 +kind: evidence +last_modified_at: 2026-06-02T18:31:17.801906+00:00 +last_modified_by: claude-code/2.1.156 +links: +- target: RFC-006 + relation: informs +status: active +title: 'RFC-006 StackExecutor live: first real harness×model patch + judged score (aider×qwen×be_01)' +--- + +## Summary + +RFC-006 StackExecutor (Half A: harness → patch) is implemented and validated +end-to-end on real money. The candidate pipeline produces a real +`harness × model × task` patch with metered cost, and the Half A→B bridge feeds +it to the judge panel for a real score. + +## Structured Fields + +- **verdict**: PASS +- **congruence_level**: CL3 (same context — this is RFC-006's own implementation, verified directly against its acceptance criteria) +- **evidence_type**: live_integration_run + +## What was verified (2026-06-02) + +1. **Network decision A (bastion) — proven, $0.** A container on the Docker + `internal` net `pollmevals-sandbox` reaches the LiteLLM proxy (HTTP 200) but + NOT the internet (DNS resolution fails). No un-metered egress; `cap_drop ALL` + holds (no NET_ADMIN needed). +2. **Half A plumbing — proven, $0.** `DockerHarnessLauncher` runs a no-model + command in the sandbox, writes to the writable `/workspace` bind, and + host-side git captures the diff (`--plumbing` check). Surfaced + fixed a + `DOCKER_HOST` discovery bug before any spend (cheap-signal-before-dear). +3. **First real patch — aider × qwen-3-14b × be_01.** `status=ok`, a 208-line + real Express JWT auth middleware written to `solution.ts`, `cost=$0.000626` + (metered via the proxy), in 1400 / out 2200 tokens, ~55s. +4. **First scored number — same stack, judged.** Half A→B bridge → judge panel + (inversion-free; the be_01 deterministic evaluators invert per EVID-027): + claude-sonnet 5.27 · gemini-3-flash 7.33 · gpt-5-mini 0.00\* · total $0.070. + Panel median 5.27/10; trustworthy 2-judge signal ≈ 6.3. + +## Defect surfaced + +\* **gpt-5-mini judge truncates** its rubric JSON at the 2048 `max_tokens` cap on +the be_01 7-criterion coding rubric → JSON parse-fail → 0.0 fallback, which +drags the panel median + Krippendorff α (α went negative). Fix is +`reasoning_effort` (cap reasoning tokens), NOT raising the cap — EVID-023 bounds +the cap by the OpenRouter HTTP-402 pre-reservation hazard. Tracked as judge +follow-up; does not block the executor. + +## Artifacts + +- Code: `apps/eval-core-py/src/orchestrator/stack_executor.py`, `stack_scoring.py` +- Image: `pollmevals-harness-aider:0.1.0` (aider-chat 0.86.2) +- Smokes: `scripts/stack_exec_live_smoke.py`, `scripts/stack_score_live_smoke.py` +- Runbook: `docs/04-runbook/14-stack-executor.md` +- Branches: `feat/stack-executor-rfc006` (Phases 1-3, PR #39), `feat/stack-scoring-rfc006` (Phase 4a) + + diff --git a/.forgeplan/rfcs/RFC-006-stack-executor-run-agent-cli-harnesses-in-the-sandbox-candidate-side-patch-trace-metered-cost.md b/.forgeplan/rfcs/RFC-006-stack-executor-run-agent-cli-harnesses-in-the-sandbox-candidate-side-patch-trace-metered-cost.md index 51836e4..5591e1a 100644 --- a/.forgeplan/rfcs/RFC-006-stack-executor-run-agent-cli-harnesses-in-the-sandbox-candidate-side-patch-trace-metered-cost.md +++ b/.forgeplan/rfcs/RFC-006-stack-executor-run-agent-cli-harnesses-in-the-sandbox-candidate-side-patch-trace-metered-cost.md @@ -119,3 +119,4 @@ real "model × harness" number**. Then widen to codex/opencode + more tasks. + From a1a96d419d6a61db05b152984f51f4a05d875556 Mon Sep 17 00:00:00 2001 From: gogocat Date: Tue, 2 Jun 2026 23:39:52 +0300 Subject: [PATCH 7/7] =?UTF-8?q?feat(executor):=20Phase=204b=20=E2=80=94=20?= =?UTF-8?q?GridRunner=20dispatch-by-stack=20(StackExecutorCaller)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC-006 Phase 4b. Makes a CLI stack usable through the EvalCaller Protocol so a real grid run dispatches it identically to raw-llm — the keystone that turns the standalone executor into part of the grid pipeline. - src/orchestrator/stack_caller.py: StackExecutorCaller (EvalCaller adapter) — resolve the stack adapter → seed a candidate snapshot → run StackExecutor → bridge the patch to an EvalResult (stack_scoring); a non-OK run → a graceful FAILED row (FR-009, never dropped). + factory providers (task prompt from task.yaml, be_01 snapshot) and default_model_alias (route → proxy alias). - grid_runner.py: optional caller_for_stack resolver — raw-llm → InspectEvalCaller, CLI stacks → StackExecutorCaller. Backward-compatible (falls back to the single caller when unset). GridRunner's judge hook scores the produced submission with no downstream special-casing. - 6 offline tests: caller OK→SCORED+submission / empty-patch→FAILED, cost/alias carry-over, dispatch routing, single-caller fallback. Gates: ruff + mypy --strict clean; full suite 706 passed. Refs: rfc-006 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/orchestrator/grid_runner.py | 15 +- .../src/orchestrator/stack_caller.py | 185 ++++++++++++++++++ apps/eval-core-py/tests/test_stack_caller.py | 167 ++++++++++++++++ 3 files changed, 365 insertions(+), 2 deletions(-) create mode 100644 apps/eval-core-py/src/orchestrator/stack_caller.py create mode 100644 apps/eval-core-py/tests/test_stack_caller.py diff --git a/apps/eval-core-py/src/orchestrator/grid_runner.py b/apps/eval-core-py/src/orchestrator/grid_runner.py index 9204a24..a348808 100644 --- a/apps/eval-core-py/src/orchestrator/grid_runner.py +++ b/apps/eval-core-py/src/orchestrator/grid_runner.py @@ -16,7 +16,7 @@ import dataclasses import logging import time -from collections.abc import Iterator +from collections.abc import Callable, Iterator from dataclasses import dataclass from datetime import UTC, datetime from decimal import Decimal @@ -217,8 +217,14 @@ def __init__( max_concurrent: int = MAX_CONCURRENT_EVALS, judge_panel: JudgePanel | None = None, judge_cost_estimate_per_eval: Decimal = Decimal("0.15"), + caller_for_stack: Callable[[str], EvalCaller] | None = None, ) -> None: self._caller = caller + # RFC-006 Phase 4b: dispatch-by-stack. When set, GridRunner picks the + # caller per stack_id (raw-llm → InspectEvalCaller, CLI stacks → + # StackExecutorCaller); falls back to the single ``caller`` otherwise so + # every pre-existing construction is unchanged. + self._caller_for_stack = caller_for_stack self._journal_writer = journal_writer self._budget_gate = budget_gate self._pricing_snapshot = pricing_snapshot @@ -289,8 +295,13 @@ async def _run_single(self, request: EvalRequest) -> EvalResult | None: with tracer.start_as_current_span("eval.run_single", attributes=span_attrs) as span: start_ns = time.monotonic_ns() + caller = ( + self._caller_for_stack(request.stack_id) + if self._caller_for_stack is not None + else self._caller + ) try: - result = await self._caller.call(request) + result = await caller.call(request) except Exception as exc: span.record_exception(exc) span.set_status(StatusCode.ERROR, str(exc)) diff --git a/apps/eval-core-py/src/orchestrator/stack_caller.py b/apps/eval-core-py/src/orchestrator/stack_caller.py new file mode 100644 index 0000000..829d4c5 --- /dev/null +++ b/apps/eval-core-py/src/orchestrator/stack_caller.py @@ -0,0 +1,185 @@ +"""StackExecutorCaller — make a CLI stack usable through the EvalCaller Protocol. + +RFC-006 Phase 4b. GridRunner dispatches one ``EvalCaller`` per (model, stack, +task, seed). ``raw-llm`` uses ``InspectEvalCaller`` (model completion); this +adapter lets every ``repository_patch`` stack (aider, …) flow through the SAME +interface: resolve the stack adapter → seed a candidate snapshot → run the +harness (StackExecutor) → bridge the patch to an ``EvalResult`` (stack_scoring). + +GridRunner's judge hook then scores the produced submission exactly as it does +for raw-llm — no special-casing downstream. + +Snapshot + prompt providers are injected so the adapter is unit-testable +offline (FakeHarnessLauncher + fakes), and the real wiring lives in +``make_stack_executor_caller``. +""" + +from __future__ import annotations + +import tempfile +from collections.abc import Callable +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path + +import yaml + +from src.contracts import ErrorClass, EvalRow, EvalStats, EvalStatus +from src.orchestrator.eval_caller import ( + EvalRequest, + EvalResult, + _make_stub_artifact_refs, + compute_eval_id, +) +from src.orchestrator.stack_executor import ( + ExecStatus, + StackAdapter, + StackExecRequest, + StackExecResult, + StackExecutor, +) +from src.orchestrator.stack_scoring import exec_result_to_eval_result + +# ExecStatus (Half A terminal) → ErrorClass (manifest error taxonomy). +_EXEC_ERROR_CLASS: dict[ExecStatus, ErrorClass] = { + ExecStatus.TIMEOUT: ErrorClass.TIMEOUT, + ExecStatus.FAILED: ErrorClass.SANDBOX_FAILURE, + ExecStatus.NO_PATCH: ErrorClass.SANDBOX_FAILURE, + ExecStatus.UNSUPPORTED: ErrorClass.SANDBOX_FAILURE, +} + + +def default_model_alias(model_id: str) -> str: + """Heuristic provider-route → proxy alias: the segment after the last '/'. + + e.g. ``openrouter/qwen/qwen-3-14b`` → ``qwen-3-14b``. The proxy + (litellm-config.yaml) keys on these short aliases. Inject a different + mapping when the route does not follow this shape. + """ + return model_id.rsplit("/", 1)[-1] + + +@dataclass +class StackExecutorCaller: + """EvalCaller adapter wrapping StackExecutor + the Half A→B bridge. + + Args: + executor: the StackExecutor (carries launcher + proxy + pricing). + stacks_root: dir holding ``/stack.yaml`` adapters. + snapshot_provider: ``(task_id, dest) -> None`` — seed the candidate + working dir (the harness edits this). Must NOT include gold/tests. + prompt_provider: ``task_id -> str`` — the task prompt for the harness. + log_dir: where the bridge writes submission artifacts. + run_hash: the run's content hash (passed to the bridge for eval_id). + model_alias_for: ``model_id -> proxy alias`` (default: last path segment). + """ + + executor: StackExecutor + stacks_root: Path + snapshot_provider: Callable[[str, Path], None] + prompt_provider: Callable[[str], str] + log_dir: Path + run_hash: str = "sha256:" + "5" * 64 + model_alias_for: Callable[[str], str] = default_model_alias + + async def call(self, request: EvalRequest) -> EvalResult: + started_at = datetime.now(UTC) + adapter = StackAdapter.from_yaml_path(self.stacks_root / request.stack_id / "stack.yaml") + + snapshot = Path(tempfile.mkdtemp(prefix=f"pollmevals-{request.stack_id}-")) + self.snapshot_provider(request.task_id, snapshot) + + exec_request = StackExecRequest( + eval_id=request.eval_id, + model_id=request.model_id, + model_alias=self.model_alias_for(request.model_id), + stack=adapter, + task_id=request.task_id, + task_prompt=self.prompt_provider(request.task_id), + repo_snapshot_dir=snapshot, + seed=request.seed, + timeout_s=request.timeout_s, + ) + exec_result = await self.executor.execute(exec_request) + + if exec_result.status is ExecStatus.OK: + return exec_result_to_eval_result( + exec_result, log_dir=self.log_dir, run_hash=self.run_hash + ) + return self._failed_result(request, exec_result, started_at) + + def _failed_result( + self, + request: EvalRequest, + exec_result: StackExecResult, + started_at: datetime, + ) -> EvalResult: + """Map a non-OK execution to a graceful FAILED EvalResult (FR-009).""" + eval_id = compute_eval_id( + self.run_hash, request.model_id, request.stack_id, request.task_id, request.seed + ) + error_class = _EXEC_ERROR_CLASS.get(exec_result.status, ErrorClass.SANDBOX_FAILURE) + completed_at = datetime.now(UTC) + row = EvalRow( + eval_id=eval_id, + model_id=request.model_id, + stack_id=request.stack_id, + task_id=request.task_id, + seed=request.seed, + status=EvalStatus.FAILED, + error_class=error_class, + error_detail=exec_result.error_detail or f"executor status={exec_result.status}", + artifact_refs=_make_stub_artifact_refs(eval_id), + stats=EvalStats( + input_tokens=exec_result.input_tokens, + output_tokens=exec_result.output_tokens, + wall_clock_ms=exec_result.wall_ms, + cost_usd=exec_result.cost_usd, + ), + started_at=started_at, + completed_at=completed_at, + ) + return EvalResult( + request=request, + eval_row=row, + exception=None, + started_at=started_at, + completed_at=completed_at, + ) + + +# --------------------------------------------------------------------------- +# Real-wiring factory + default providers (repo-path aware) +# --------------------------------------------------------------------------- + + +def make_task_prompt_provider(repo_root: Path) -> Callable[[str], str]: + """Read ``evals/task-packs//task.yaml`` → ``prompt_template``.""" + + def _provider(task_id: str) -> str: + pack = repo_root / "evals" / "task-packs" / task_id / "task.yaml" + data = yaml.safe_load(pack.read_text(encoding="utf-8")) + return str(data["prompt_template"]) + + return _provider + + +def make_be01_snapshot_provider(repo_root: Path) -> Callable[[str, Path], None]: + """Seed a be_01 candidate workspace: pinned deps only — NO gold, NO tests. + + Other tasks need their own provider; this is the first-slice default. A + future convention (a ``candidate/`` scaffold dir per pack) generalises it. + """ + gold = repo_root / "evals" / "task-packs" / "be_01_jwt_auth" / "gold" + + def _provider(task_id: str, dest: Path) -> None: + import shutil + + for f in ("package.json", "tsconfig.json"): + shutil.copy(gold / f, dest / f) + (dest / "solution.ts").write_text( + "// Implement the Express JWT auth middleware here (see the task prompt).\n", + encoding="utf-8", + ) + + return _provider diff --git a/apps/eval-core-py/tests/test_stack_caller.py b/apps/eval-core-py/tests/test_stack_caller.py new file mode 100644 index 0000000..793181c --- /dev/null +++ b/apps/eval-core-py/tests/test_stack_caller.py @@ -0,0 +1,167 @@ +"""Unit tests for StackExecutorCaller + GridRunner dispatch-by-stack (Phase 4b). + +Offline — FakeHarnessLauncher + FakeEvalCaller, no Docker / judges / network. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from decimal import Decimal +from pathlib import Path + +import pytest + +from src.contracts import EvalStatus +from src.orchestrator.cost import BudgetGate +from src.orchestrator.eval_caller import EvalRequest, EvalResult, FakeEvalCaller +from src.orchestrator.grid_runner import GridRunner, GridSpec +from src.orchestrator.journal import JournalWriter +from src.orchestrator.stack_caller import StackExecutorCaller, default_model_alias +from src.orchestrator.stack_executor import ( + FakeHarnessLauncher, + HarnessRunOutcome, + StackExecutor, +) + +_AIDER_STACK_YAML = """ +slug: aider +agent_cli: aider +execution: + mode: repository_patch + command: aider + args: [--yes] +limits: + max_wall_clock_seconds: 600 +""" + +_PATCH = ( + "diff --git a/solution.ts b/solution.ts\n" + "--- a/solution.ts\n+++ b/solution.ts\n@@ -1 +1 @@\n+export const ok = 1;\n" +) + + +def _stacks_root(tmp_path: Path) -> Path: + root = tmp_path / "stacks" + (root / "aider").mkdir(parents=True) + (root / "aider" / "stack.yaml").write_text(_AIDER_STACK_YAML) + return root + + +def _seed_solution(task_id: str, dest: Path) -> None: + (dest / "solution.ts").write_text("export const ok = 1;\n") + + +def _caller(tmp_path: Path, *, outcome: HarnessRunOutcome | None = None) -> StackExecutorCaller: + launcher = FakeHarnessLauncher(outcome=outcome) if outcome else FakeHarnessLauncher() + return StackExecutorCaller( + executor=StackExecutor(launcher=launcher), + stacks_root=_stacks_root(tmp_path), + snapshot_provider=_seed_solution, + prompt_provider=lambda t: "implement it", + log_dir=tmp_path / "artifacts", + ) + + +def _request(stack_id: str = "aider") -> EvalRequest: + return EvalRequest( + eval_id="abcdef0123456789", + model_id="openrouter/qwen/qwen-3-14b", + stack_id=stack_id, + task_id="be_01_jwt_auth", + seed=1, + ) + + +class TestStackExecutorCaller: + @pytest.mark.asyncio + async def test_ok_run_produces_scored_eval_result(self, tmp_path: Path) -> None: + outcome = HarnessRunOutcome(0, _PATCH, "trace", "", 100, 50, 0, 1000, False) + result = await _caller(tmp_path, outcome=outcome).call(_request()) + assert result.eval_row is not None + assert result.eval_row.status is EvalStatus.SCORED + assert result.eval_row.stack_id == "aider" + # submission artifact carries the changed solution.ts content + uri = result.eval_row.artifact_refs.raw_output.uri + assert Path(uri[len("file://") :]).read_text().find("export const ok = 1;") != -1 + + @pytest.mark.asyncio + async def test_no_patch_run_is_failed_not_dropped(self, tmp_path: Path) -> None: + outcome = HarnessRunOutcome(0, "", "trace", "", 10, 5, 0, 100, False) # empty patch + result = await _caller(tmp_path, outcome=outcome).call(_request()) + assert result.eval_row is not None + assert result.eval_row.status is EvalStatus.FAILED + assert result.eval_row.error_class is not None + + @pytest.mark.asyncio + async def test_carries_cost_and_alias(self, tmp_path: Path) -> None: + outcome = HarnessRunOutcome(0, _PATCH, "t", "", 1000, 500, 0, 2000, False) + # no pricing snapshot on the executor → cost 0, but tokens carry over + result = await _caller(tmp_path, outcome=outcome).call(_request()) + assert result.eval_row is not None + assert result.eval_row.stats.input_tokens == 1000 + + def test_default_model_alias(self) -> None: + assert default_model_alias("openrouter/qwen/qwen-3-14b") == "qwen-3-14b" + assert default_model_alias("qwen-3-14b") == "qwen-3-14b" + + +@dataclass +class _SpyCaller: + """Records the stack_ids it was asked to run; delegates to FakeEvalCaller.""" + + seen: list[str] = field(default_factory=list) + inner: FakeEvalCaller = field(default_factory=FakeEvalCaller) + + async def call(self, request: EvalRequest) -> EvalResult: + self.seen.append(request.stack_id) + return await self.inner.call(request) + + +class TestGridRunnerDispatchByStack: + @pytest.mark.asyncio + async def test_routes_caller_per_stack(self, tmp_path: Path) -> None: + writer = JournalWriter(tmp_path / "j.ndjson") + raw_spy, cli_spy = _SpyCaller(), _SpyCaller() + + def caller_for(stack_id: str) -> _SpyCaller: + return cli_spy if stack_id == "aider" else raw_spy + + runner = GridRunner( + caller=raw_spy, # default (unused when caller_for_stack is set) + caller_for_stack=caller_for, + journal_writer=writer, + budget_gate=BudgetGate(cap_usd=Decimal("9999")), + pricing_snapshot={}, + ) + spec = GridSpec( + run_hash="sha256:" + "a" * 64, + models=["m"], + tasks=["be_01_jwt_auth"], + stacks=["raw-llm", "aider"], + seeds=[1], + ) + await runner.run(spec) + + assert cli_spy.seen == ["aider"] + assert raw_spy.seen == ["raw-llm"] + + @pytest.mark.asyncio + async def test_falls_back_to_single_caller(self, tmp_path: Path) -> None: + # No caller_for_stack → every stack uses the single caller (back-compat). + writer = JournalWriter(tmp_path / "j.ndjson") + spy = _SpyCaller() + runner = GridRunner( + caller=spy, + journal_writer=writer, + budget_gate=BudgetGate(cap_usd=Decimal("9999")), + pricing_snapshot={}, + ) + spec = GridSpec( + run_hash="sha256:" + "b" * 64, + models=["m"], + tasks=["t"], + stacks=["raw-llm", "aider"], + seeds=[1], + ) + await runner.run(spec) + assert sorted(spy.seen) == ["aider", "raw-llm"]