feat: Terminal-Bench adapter for the release canary (release evaluation PR 2)

.gitignore

@@ -8,3 +8,4 @@ packages/harness/assets/
 
 # Generated eval run evidence
 evals/jobs/
+__pycache__/

evals/external/terminal-bench/agent.mjs

@@ -0,0 +1,150 @@
+/**
+ * Stdio bridge agent: the Node side of the Harbor custom-agent integration.
+ *
+ * Harbor custom agents are Python classes, but the release evaluation's
+ * decision-making stack (driver, profiles, budget, telemetry) lives in Node.
+ * The bridge keeps one process on each side of a line-delimited JSON
+ * protocol:
+ *
+ *   Node → Python:  {type:'exec', id, command, timeoutMs}
+ *                   {type:'done', answer, stopReason, steps, telemetry, ...}
+ *   Python → Node:  {type:'result', id, code, stdout, stderr}
+ *
+ * The Python wrapper (`harbor_agent.py`) executes each `exec` inside the
+ * Harbor environment and pumps the result back; every provider decision,
+ * budget precheck, and telemetry event stays in Node where it is tested.
+ */
+import fs from 'node:fs';
+import readline from 'node:readline';
+import { pathToFileURL } from 'node:url';
+import { openAiToolDriver } from '../../lib/drivers.mjs';
+import { getProfile } from '../../lib/model-profiles.mjs';
+import { createBudget } from '../../lib/budget.mjs';
+import { createTelemetry } from '../../lib/telemetry.mjs';
+
+export const BRIDGE_TOOLS = [
+  {
+    name: 'bash',
+    description: 'Run a shell command in the task environment and return its exit code and output.',
+    parameters: { type: 'object', properties: { command: { type: 'string' } }, required: ['command'] },
+  },
+  {
+    name: 'finish',
+    description: 'End the task with a final answer once the work is complete and verified.',
+    parameters: { type: 'object', properties: { answer: { type: 'string' } }, required: ['answer'] },
+  },
+];
+
+/**
+ * Drive the loop over stdio streams. Returns the final `done` payload; every
+ * exit path (finish, provider error, step ceiling, protocol error) reports an
+ * explicit stopReason and carries the telemetry snapshot when available.
+ */
+export async function runStdioAgent({
+  driver,
+  input,
+  output,
+  systemPrompt,
+  instruction,
+  maxSteps = 50,
+  telemetry = null,
+  execTimeoutMs = 120_000,
+}) {
+  driver.reset?.({ system: systemPrompt, instruction, tools: BRIDGE_TOOLS });
+
+  const rl = readline.createInterface({ input });
+  const pendingLines = [];
+  const waiters = [];
+  rl.on('line', (line) => {
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      parsed = { type: 'protocol_error', raw: line.slice(0, 200) };
+    }
+    const waiter = waiters.shift();
+    if (waiter) waiter(parsed);
+    else pendingLines.push(parsed);
+  });
+  const nextLine = () =>
+    new Promise((resolve) => {
+      if (pendingLines.length) resolve(pendingLines.shift());
+      else waiters.push(resolve);
+    });
+  const send = (msg) => output.write(`${JSON.stringify(msg)}\n`);
+
+  let execId = 0;
+  let steps = 0;
+  const finish = (payload) => {
+    const done = { type: 'done', steps, telemetry: telemetry?.snapshot() ?? null, ...payload };
+    send(done);
+    rl.close();
+    return done;
+  };
+
+  while (steps < maxSteps) {
+    let action;
+    try {
+      action = await driver.next();
+    } catch (err) {
+      return finish({
+        answer: null,
+        stopReason: 'provider_error',
+        providerFailure: { kind: err.kind ?? 'unknown', billed: err.billed ?? null, message: err.message },
+      });
+    }
+    if (!action || action.type === 'finish') {
+      return finish({ answer: action?.answer ?? null, stopReason: action?.stopReason ?? 'model_finish' });
+    }
+    steps += 1;
+    if (action.name !== 'bash') {
+      driver.observe?.(action, { error: `unknown tool: ${action.name}` });
+      continue;
+    }
+    const id = execId++;
+    send({ type: 'exec', id, command: action.input?.command ?? '', timeoutMs: execTimeoutMs });
+    const result = await nextLine();
+    if (result.type !== 'result' || result.id !== id) {
+      return finish({ answer: null, stopReason: 'protocol_error', detail: JSON.stringify(result).slice(0, 200) });
+    }
+    driver.observe?.(action, { code: result.code, stdout: result.stdout, stderr: result.stderr });
+  }
+  return finish({ answer: null, stopReason: 'max_steps' });
+}
+
+/** CLI entry used by harbor_agent.py: node agent.mjs --condition <file> [--instruction <file>] */
+async function main() {
+  const args = process.argv.slice(2);
+  const flag = (name) => {
+    const i = args.indexOf(name);
+    return i >= 0 ? args[i + 1] : null;
+  };
+  const condition = JSON.parse(fs.readFileSync(flag('--condition'), 'utf8'));
+  const instructionPath = flag('--instruction');
+  const instruction = instructionPath ? fs.readFileSync(instructionPath, 'utf8') : condition.instruction;
+  const profile = getProfile(condition.profileId ?? 'kimi-k2.7-code');
+  const apiKey = process.env[condition.apiKeyEnv ?? 'OPENROUTER_API_KEY'] ?? 'local';
+  const telemetry = createTelemetry();
+  const budget = createBudget({
+    ceilingUsd: condition.limits?.trialCeilingUsd ?? profile.trialCeilingUsd,
+    label: `${condition.id}-trial`,
+  });
+  const driver = openAiToolDriver({ profile, apiKey, budget, telemetry, maxTokens: condition.limits?.maxOutputTokens });
+  if (!driver) throw new Error('driver not configured: check profile and API key environment');
+  await runStdioAgent({
+    driver,
+    input: process.stdin,
+    output: process.stdout,
+    systemPrompt: condition.systemPrompt,
+    instruction,
+    maxSteps: condition.limits?.maxSteps ?? 50,
+    telemetry,
+  });
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  main().catch((err) => {
+    process.stdout.write(`${JSON.stringify({ type: 'done', answer: null, stopReason: 'bridge_error', detail: err.message })}\n`);
+    process.exit(1);
+  });
+}

evals/external/terminal-bench/generic-condition.mjs

@@ -0,0 +1,30 @@
+/**
+ * Generic condition: the untreated baseline of the A/B pair.
+ *
+ * The baseline gets the original Terminal-Bench instruction and a neutral,
+ * competent software-engineering prompt — deliberately fair (it encourages
+ * exploring, testing, and verifying) but with zero Engineer Harness workflow:
+ * no contract, no loaded-skill guidance, no activation commands. Wording here
+ * is checked by tests to keep harness vocabulary from leaking into the
+ * control arm.
+ */
+
+export const NEUTRAL_SYSTEM_PROMPT = [
+  'You are an experienced software engineer working in a Linux terminal.',
+  'Complete the task exactly as instructed.',
+  'Explore the environment first to understand the code and data you are working with.',
+  'Make focused changes, run the relevant commands and tests, and verify your outputs meet the requirements before you finish.',
+  'Prefer small, checkable steps over large speculative changes.',
+].join(' ');
+
+export function buildGenericCondition({ instruction, limits } = {}) {
+  if (!instruction) throw new Error('instruction is required');
+  if (!limits) throw new Error('limits is required');
+  return {
+    id: 'generic',
+    systemPrompt: NEUTRAL_SYSTEM_PROMPT,
+    instruction,
+    setupCommands: [],
+    limits: { ...limits },
+  };
+}

evals/external/terminal-bench/harbor-adapter.mjs

@@ -0,0 +1,109 @@
+/**
+ * Harbor CLI adapter for the pinned Terminal-Bench release canary.
+ *
+ * The plan's rule is to use Harbor rather than re-implement Terminal-Bench
+ * execution, so this module owns exactly the seams around the `harbor` CLI:
+ *
+ *   - task pinning: validate `task-lock.json`, stamp/verify the task tree
+ *     checksum so a drifted or tampered task fails closed before any spend;
+ *   - command construction: only flags evidenced by the Harbor docs
+ *     (`run -d <dataset@version> --task-name <task> --agent <ref> --model
+ *     <m> --env <docker|daytona> -n 1`);
+ *   - process execution behind an injected spawn (deterministic in CI);
+ *   - job/trial result discovery via the verifier evidence reader;
+ *   - failure classification: infrastructure vs provider vs verifier vs a
+ *     graded trial (a reward of 0 is a fail, not a failure).
+ *
+ * Jobs land in `<cwd>/jobs`, so callers control output placement by cwd
+ * rather than by version-fragile CLI flags.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { collectVerifierEvidence, hashTree, verdictFromReward } from './verifier.mjs';
+
+const REQUIRED_LOCK_FIELDS = ['lockSchema', 'datasetRef', 'task', 'verifier'];
+
+export function validateTaskLock(lock) {
+  const errors = [];
+  for (const field of REQUIRED_LOCK_FIELDS) {
+    if (lock?.[field] == null) errors.push(`missing required lock field: ${field}`);
+  }
+  if (lock?.datasetRef != null && !/^[\w./-]+@[\w.-]+$/.test(lock.datasetRef)) {
+    errors.push(`datasetRef must pin a version (name@version), got: ${lock.datasetRef}`);
+  }
+  if (lock?.verifier != null && typeof lock.verifier.passingReward !== 'number') {
+    errors.push('verifier.passingReward must be a number');
+  }
+  return { ok: errors.length === 0, errors };
+}
+
+/** Return a copy of the lock pinned to the given task directory's checksum. */
+export function stampTaskLock(taskDir, lock) {
+  return { ...lock, taskChecksum: hashTree(taskDir) };
+}
+
+/** Fail closed: an unstamped lock or a drifted task tree both refuse the run. */
+export function verifyTaskAgainstLock(taskDir, lock) {
+  const structural = validateTaskLock(lock);
+  if (!structural.ok) return { ok: false, reason: structural.errors.join('; '), checksum: null };
+  if (!lock.taskChecksum) {
+    return { ok: false, reason: 'task lock is not stamped (taskChecksum is null) — run stampTaskLock against the pinned task', checksum: null };
+  }
+  const checksum = hashTree(taskDir);
+  if (checksum !== lock.taskChecksum) {
+    return { ok: false, reason: `task checksum mismatch: expected ${lock.taskChecksum}, got ${checksum}`, checksum };
+  }
+  return { ok: true, reason: '', checksum };
+}
+
+export function buildHarborRunArgs({ lock, agentRef, model, envName, trials = 1 }) {
+  return ['run', '-d', lock.datasetRef, '--task-name', lock.task, '--agent', agentRef, '--model', model, '--env', envName, '-n', String(trials)];
+}
+
+/** Run the harbor CLI. `spawnImpl` mirrors spawnSync's contract for testability. */
+export function runHarbor({ args, cwd, spawnImpl = spawnSync, timeoutMs }) {
+  const res = spawnImpl('harbor', args, { cwd, encoding: 'utf8', timeout: timeoutMs });
+  return {
+    code: res.status ?? null,
+    stdout: res.stdout || '',
+    stderr: res.stderr || '',
+    timedOut: res.error?.code === 'ETIMEDOUT',
+    spawnError: res.error && res.error.code !== 'ETIMEDOUT' ? res.error.code || res.error.message : null,
+  };
+}
+
+/** Newest job directory under `<cwd>/jobs`, by mtime then name. */
+export function findLatestJobDir(jobsRoot) {
+  let entries;
+  try {
+    entries = fs.readdirSync(jobsRoot, { withFileTypes: true });
+  } catch {
+    return null;
+  }
+  const dirs = entries
+    .filter((e) => e.isDirectory())
+    .map((e) => {
+      const full = path.join(jobsRoot, e.name);
+      return { full, name: e.name, mtimeMs: fs.statSync(full).mtimeMs };
+    })
+    .sort((a, b) => a.mtimeMs - b.mtimeMs || (a.name < b.name ? -1 : 1));
+  return dirs.at(-1)?.full ?? null;
+}
+
+/** Verifier evidence + verdict for the (single) trial inside a job directory. */
+export function readTrialResult(jobDir, { passingReward = 1 } = {}) {
+  const evidence = collectVerifierEvidence(jobDir);
+  return { ...evidence, verdict: verdictFromReward(evidence.reward, { passingReward }) };
+}
+
+/**
+ * Classify a completed run. Returns the failure kind, or null for a valid
+ * graded trial (whose pass/fail comes from the reward, not from here).
+ */
+export function classifyFailure({ run, reward, providerFailure = false }) {
+  if (run.spawnError || run.timedOut) return 'infrastructure';
+  if (providerFailure) return 'provider';
+  if (reward == null) return 'verifier';
+  return null;
+}