diff --git a/.gitignore b/.gitignore index 480cfeef..62f68bb1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ packages/harness/assets/ # Generated eval run evidence evals/jobs/ +__pycache__/ diff --git a/evals/external/terminal-bench/agent.mjs b/evals/external/terminal-bench/agent.mjs new file mode 100644 index 00000000..210d7296 --- /dev/null +++ b/evals/external/terminal-bench/agent.mjs @@ -0,0 +1,150 @@ +/** + * Stdio bridge agent: the Node side of the Harbor custom-agent integration. + * + * Harbor custom agents are Python classes, but the release evaluation's + * decision-making stack (driver, profiles, budget, telemetry) lives in Node. + * The bridge keeps one process on each side of a line-delimited JSON + * protocol: + * + * Node → Python: {type:'exec', id, command, timeoutMs} + * {type:'done', answer, stopReason, steps, telemetry, ...} + * Python → Node: {type:'result', id, code, stdout, stderr} + * + * The Python wrapper (`harbor_agent.py`) executes each `exec` inside the + * Harbor environment and pumps the result back; every provider decision, + * budget precheck, and telemetry event stays in Node where it is tested. + */ +import fs from 'node:fs'; +import readline from 'node:readline'; +import { pathToFileURL } from 'node:url'; +import { openAiToolDriver } from '../../lib/drivers.mjs'; +import { getProfile } from '../../lib/model-profiles.mjs'; +import { createBudget } from '../../lib/budget.mjs'; +import { createTelemetry } from '../../lib/telemetry.mjs'; + +export const BRIDGE_TOOLS = [ + { + name: 'bash', + description: 'Run a shell command in the task environment and return its exit code and output.', + parameters: { type: 'object', properties: { command: { type: 'string' } }, required: ['command'] }, + }, + { + name: 'finish', + description: 'End the task with a final answer once the work is complete and verified.', + parameters: { type: 'object', properties: { answer: { type: 'string' } }, required: ['answer'] }, + }, +]; + +/** + * Drive the loop over stdio streams. Returns the final `done` payload; every + * exit path (finish, provider error, step ceiling, protocol error) reports an + * explicit stopReason and carries the telemetry snapshot when available. + */ +export async function runStdioAgent({ + driver, + input, + output, + systemPrompt, + instruction, + maxSteps = 50, + telemetry = null, + execTimeoutMs = 120_000, +}) { + driver.reset?.({ system: systemPrompt, instruction, tools: BRIDGE_TOOLS }); + + const rl = readline.createInterface({ input }); + const pendingLines = []; + const waiters = []; + rl.on('line', (line) => { + let parsed; + try { + parsed = JSON.parse(line); + } catch { + parsed = { type: 'protocol_error', raw: line.slice(0, 200) }; + } + const waiter = waiters.shift(); + if (waiter) waiter(parsed); + else pendingLines.push(parsed); + }); + const nextLine = () => + new Promise((resolve) => { + if (pendingLines.length) resolve(pendingLines.shift()); + else waiters.push(resolve); + }); + const send = (msg) => output.write(`${JSON.stringify(msg)}\n`); + + let execId = 0; + let steps = 0; + const finish = (payload) => { + const done = { type: 'done', steps, telemetry: telemetry?.snapshot() ?? null, ...payload }; + send(done); + rl.close(); + return done; + }; + + while (steps < maxSteps) { + let action; + try { + action = await driver.next(); + } catch (err) { + return finish({ + answer: null, + stopReason: 'provider_error', + providerFailure: { kind: err.kind ?? 'unknown', billed: err.billed ?? null, message: err.message }, + }); + } + if (!action || action.type === 'finish') { + return finish({ answer: action?.answer ?? null, stopReason: action?.stopReason ?? 'model_finish' }); + } + steps += 1; + if (action.name !== 'bash') { + driver.observe?.(action, { error: `unknown tool: ${action.name}` }); + continue; + } + const id = execId++; + send({ type: 'exec', id, command: action.input?.command ?? '', timeoutMs: execTimeoutMs }); + const result = await nextLine(); + if (result.type !== 'result' || result.id !== id) { + return finish({ answer: null, stopReason: 'protocol_error', detail: JSON.stringify(result).slice(0, 200) }); + } + driver.observe?.(action, { code: result.code, stdout: result.stdout, stderr: result.stderr }); + } + return finish({ answer: null, stopReason: 'max_steps' }); +} + +/** CLI entry used by harbor_agent.py: node agent.mjs --condition [--instruction ] */ +async function main() { + const args = process.argv.slice(2); + const flag = (name) => { + const i = args.indexOf(name); + return i >= 0 ? args[i + 1] : null; + }; + const condition = JSON.parse(fs.readFileSync(flag('--condition'), 'utf8')); + const instructionPath = flag('--instruction'); + const instruction = instructionPath ? fs.readFileSync(instructionPath, 'utf8') : condition.instruction; + const profile = getProfile(condition.profileId ?? 'kimi-k2.7-code'); + const apiKey = process.env[condition.apiKeyEnv ?? 'OPENROUTER_API_KEY'] ?? 'local'; + const telemetry = createTelemetry(); + const budget = createBudget({ + ceilingUsd: condition.limits?.trialCeilingUsd ?? profile.trialCeilingUsd, + label: `${condition.id}-trial`, + }); + const driver = openAiToolDriver({ profile, apiKey, budget, telemetry, maxTokens: condition.limits?.maxOutputTokens }); + if (!driver) throw new Error('driver not configured: check profile and API key environment'); + await runStdioAgent({ + driver, + input: process.stdin, + output: process.stdout, + systemPrompt: condition.systemPrompt, + instruction, + maxSteps: condition.limits?.maxSteps ?? 50, + telemetry, + }); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((err) => { + process.stdout.write(`${JSON.stringify({ type: 'done', answer: null, stopReason: 'bridge_error', detail: err.message })}\n`); + process.exit(1); + }); +} diff --git a/evals/external/terminal-bench/generic-condition.mjs b/evals/external/terminal-bench/generic-condition.mjs new file mode 100644 index 00000000..fb00154d --- /dev/null +++ b/evals/external/terminal-bench/generic-condition.mjs @@ -0,0 +1,30 @@ +/** + * Generic condition: the untreated baseline of the A/B pair. + * + * The baseline gets the original Terminal-Bench instruction and a neutral, + * competent software-engineering prompt — deliberately fair (it encourages + * exploring, testing, and verifying) but with zero Engineer Harness workflow: + * no contract, no loaded-skill guidance, no activation commands. Wording here + * is checked by tests to keep harness vocabulary from leaking into the + * control arm. + */ + +export const NEUTRAL_SYSTEM_PROMPT = [ + 'You are an experienced software engineer working in a Linux terminal.', + 'Complete the task exactly as instructed.', + 'Explore the environment first to understand the code and data you are working with.', + 'Make focused changes, run the relevant commands and tests, and verify your outputs meet the requirements before you finish.', + 'Prefer small, checkable steps over large speculative changes.', +].join(' '); + +export function buildGenericCondition({ instruction, limits } = {}) { + if (!instruction) throw new Error('instruction is required'); + if (!limits) throw new Error('limits is required'); + return { + id: 'generic', + systemPrompt: NEUTRAL_SYSTEM_PROMPT, + instruction, + setupCommands: [], + limits: { ...limits }, + }; +} diff --git a/evals/external/terminal-bench/harbor-adapter.mjs b/evals/external/terminal-bench/harbor-adapter.mjs new file mode 100644 index 00000000..8243e04d --- /dev/null +++ b/evals/external/terminal-bench/harbor-adapter.mjs @@ -0,0 +1,109 @@ +/** + * Harbor CLI adapter for the pinned Terminal-Bench release canary. + * + * The plan's rule is to use Harbor rather than re-implement Terminal-Bench + * execution, so this module owns exactly the seams around the `harbor` CLI: + * + * - task pinning: validate `task-lock.json`, stamp/verify the task tree + * checksum so a drifted or tampered task fails closed before any spend; + * - command construction: only flags evidenced by the Harbor docs + * (`run -d --task-name --agent --model + * --env -n 1`); + * - process execution behind an injected spawn (deterministic in CI); + * - job/trial result discovery via the verifier evidence reader; + * - failure classification: infrastructure vs provider vs verifier vs a + * graded trial (a reward of 0 is a fail, not a failure). + * + * Jobs land in `/jobs`, so callers control output placement by cwd + * rather than by version-fragile CLI flags. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { collectVerifierEvidence, hashTree, verdictFromReward } from './verifier.mjs'; + +const REQUIRED_LOCK_FIELDS = ['lockSchema', 'datasetRef', 'task', 'verifier']; + +export function validateTaskLock(lock) { + const errors = []; + for (const field of REQUIRED_LOCK_FIELDS) { + if (lock?.[field] == null) errors.push(`missing required lock field: ${field}`); + } + if (lock?.datasetRef != null && !/^[\w./-]+@[\w.-]+$/.test(lock.datasetRef)) { + errors.push(`datasetRef must pin a version (name@version), got: ${lock.datasetRef}`); + } + if (lock?.verifier != null && typeof lock.verifier.passingReward !== 'number') { + errors.push('verifier.passingReward must be a number'); + } + return { ok: errors.length === 0, errors }; +} + +/** Return a copy of the lock pinned to the given task directory's checksum. */ +export function stampTaskLock(taskDir, lock) { + return { ...lock, taskChecksum: hashTree(taskDir) }; +} + +/** Fail closed: an unstamped lock or a drifted task tree both refuse the run. */ +export function verifyTaskAgainstLock(taskDir, lock) { + const structural = validateTaskLock(lock); + if (!structural.ok) return { ok: false, reason: structural.errors.join('; '), checksum: null }; + if (!lock.taskChecksum) { + return { ok: false, reason: 'task lock is not stamped (taskChecksum is null) — run stampTaskLock against the pinned task', checksum: null }; + } + const checksum = hashTree(taskDir); + if (checksum !== lock.taskChecksum) { + return { ok: false, reason: `task checksum mismatch: expected ${lock.taskChecksum}, got ${checksum}`, checksum }; + } + return { ok: true, reason: '', checksum }; +} + +export function buildHarborRunArgs({ lock, agentRef, model, envName, trials = 1 }) { + return ['run', '-d', lock.datasetRef, '--task-name', lock.task, '--agent', agentRef, '--model', model, '--env', envName, '-n', String(trials)]; +} + +/** Run the harbor CLI. `spawnImpl` mirrors spawnSync's contract for testability. */ +export function runHarbor({ args, cwd, spawnImpl = spawnSync, timeoutMs }) { + const res = spawnImpl('harbor', args, { cwd, encoding: 'utf8', timeout: timeoutMs }); + return { + code: res.status ?? null, + stdout: res.stdout || '', + stderr: res.stderr || '', + timedOut: res.error?.code === 'ETIMEDOUT', + spawnError: res.error && res.error.code !== 'ETIMEDOUT' ? res.error.code || res.error.message : null, + }; +} + +/** Newest job directory under `/jobs`, by mtime then name. */ +export function findLatestJobDir(jobsRoot) { + let entries; + try { + entries = fs.readdirSync(jobsRoot, { withFileTypes: true }); + } catch { + return null; + } + const dirs = entries + .filter((e) => e.isDirectory()) + .map((e) => { + const full = path.join(jobsRoot, e.name); + return { full, name: e.name, mtimeMs: fs.statSync(full).mtimeMs }; + }) + .sort((a, b) => a.mtimeMs - b.mtimeMs || (a.name < b.name ? -1 : 1)); + return dirs.at(-1)?.full ?? null; +} + +/** Verifier evidence + verdict for the (single) trial inside a job directory. */ +export function readTrialResult(jobDir, { passingReward = 1 } = {}) { + const evidence = collectVerifierEvidence(jobDir); + return { ...evidence, verdict: verdictFromReward(evidence.reward, { passingReward }) }; +} + +/** + * Classify a completed run. Returns the failure kind, or null for a valid + * graded trial (whose pass/fail comes from the reward, not from here). + */ +export function classifyFailure({ run, reward, providerFailure = false }) { + if (run.spawnError || run.timedOut) return 'infrastructure'; + if (providerFailure) return 'provider'; + if (reward == null) return 'verifier'; + return null; +} diff --git a/evals/external/terminal-bench/harbor_agent.py b/evals/external/terminal-bench/harbor_agent.py new file mode 100644 index 00000000..b5fd5537 --- /dev/null +++ b/evals/external/terminal-bench/harbor_agent.py @@ -0,0 +1,117 @@ +"""Harbor external agent that bridges to the Node stdio agent (agent.mjs). + +Harbor invokes this class via: + + harbor run -d terminal-bench@2.0 --task-name cobol-modernization \ + --agent evals.external.terminal_bench.harbor_agent:StdioBridgeAgent ... + +All decision-making (model driver, budget prechecks, telemetry) happens in the +Node process; this wrapper only executes each requested command inside the +Harbor environment and pumps results back over the line-delimited JSON +protocol documented in agent.mjs. + +Configuration comes from environment variables set by the release runner: + + HARNESS_EVAL_TB_CONDITION path to the condition JSON (required) + HARNESS_EVAL_TB_NODE node binary (default: "node") + HARNESS_EVAL_TB_AGENT_MJS path to agent.mjs (default: alongside this file) + +The exact BaseEnvironment exec surface can differ between Harbor releases, so +`_exec` resolves the method defensively and normalizes the result shape. This +wrapper is exercised for real at release time; repository tests cover the Node +side of the protocol. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import pathlib + +from harbor.agents.base import BaseAgent + + +class StdioBridgeAgent(BaseAgent): + @staticmethod + def name() -> str: + return "engineer-harness-stdio-bridge" + + def version(self) -> str | None: + return "1" + + async def setup(self, environment) -> None: + condition = self._load_condition() + for command in condition.get("setupCommands", []): + await self._exec(environment, command) + + async def run(self, instruction: str, environment, context) -> None: + condition_path = os.environ["HARNESS_EVAL_TB_CONDITION"] + node = os.environ.get("HARNESS_EVAL_TB_NODE", "node") + agent_mjs = os.environ.get( + "HARNESS_EVAL_TB_AGENT_MJS", + str(pathlib.Path(__file__).with_name("agent.mjs")), + ) + instruction_path = pathlib.Path(condition_path).with_suffix(".instruction.txt") + instruction_path.write_text(instruction) + + proc = await asyncio.create_subprocess_exec( + node, + agent_mjs, + "--condition", + condition_path, + "--instruction", + str(instruction_path), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + env=os.environ.copy(), + ) + try: + while True: + line = await proc.stdout.readline() + if not line: + break + message = json.loads(line) + if message["type"] == "exec": + result = await self._exec(environment, message["command"]) + reply = {"type": "result", "id": message["id"], **result} + proc.stdin.write((json.dumps(reply) + "\n").encode()) + await proc.stdin.drain() + elif message["type"] == "done": + self._populate_context(context, message) + break + finally: + if proc.returncode is None: + proc.terminate() + await proc.wait() + + def _load_condition(self) -> dict: + with open(os.environ["HARNESS_EVAL_TB_CONDITION"]) as fh: + return json.load(fh) + + async def _exec(self, environment, command: str) -> dict: + """Execute a command via whichever exec surface this Harbor exposes.""" + exec_fn = getattr(environment, "exec", None) or getattr(environment, "execute", None) + if exec_fn is None: + raise RuntimeError("Harbor environment exposes no exec/execute method") + result = await exec_fn(command=command) + code = getattr(result, "return_code", None) + if code is None: + code = getattr(result, "exit_code", 0) + stdout = getattr(result, "output", None) + if stdout is None: + stdout = getattr(result, "stdout", "") + stderr = getattr(result, "stderr", "") or "" + return {"code": code, "stdout": (stdout or "")[-6000:], "stderr": stderr[-2000:]} + + def _populate_context(self, context, done: dict) -> None: + """Attach the bridge outcome to whatever context fields this Harbor has.""" + for attr, value in ( + ("final_answer", done.get("answer")), + ("stop_reason", done.get("stopReason")), + ("metadata", {"telemetry": done.get("telemetry"), "steps": done.get("steps")}), + ): + try: + setattr(context, attr, value) + except (AttributeError, TypeError): + pass diff --git a/evals/external/terminal-bench/harness-condition.mjs b/evals/external/terminal-bench/harness-condition.mjs new file mode 100644 index 00000000..3f104460 --- /dev/null +++ b/evals/external/terminal-bench/harness-condition.mjs @@ -0,0 +1,27 @@ +/** + * Harness condition: the treatment arm of the A/B pair. + * + * Identical baseline (same neutral prompt, same instruction, same limits) + * plus everything the Engineer Harness adds: the engineer agent contract, + * loaded-skill guidance, and CLI activation commands run at sandbox setup. + * The added context and setup cost are deliberately charged to this arm — + * they are real product overhead. + */ +import { NEUTRAL_SYSTEM_PROMPT } from './generic-condition.mjs'; + +const DEFAULT_ACTIVATION = ['harness install']; + +export function buildHarnessCondition({ instruction, limits, engineerContract, guidance = '', activationCommands = DEFAULT_ACTIVATION } = {}) { + if (!instruction) throw new Error('instruction is required'); + if (!limits) throw new Error('limits is required'); + if (!engineerContract) throw new Error('engineerContract is required'); + const sections = [NEUTRAL_SYSTEM_PROMPT, engineerContract]; + if (guidance) sections.push(guidance); + return { + id: 'harness', + systemPrompt: sections.join('\n\n'), + instruction, + setupCommands: [...activationCommands], + limits: { ...limits }, + }; +} diff --git a/evals/external/terminal-bench/task-lock.json b/evals/external/terminal-bench/task-lock.json new file mode 100644 index 00000000..663b74dd --- /dev/null +++ b/evals/external/terminal-bench/task-lock.json @@ -0,0 +1,12 @@ +{ + "lockSchema": 1, + "datasetRef": "terminal-bench@2.0", + "task": "cobol-modernization", + "registryUrl": "https://www.tbench.ai/benchmarks/terminal-bench-2/cobol-modernization", + "lockedAt": "2026-07-30", + "taskChecksum": null, + "verifier": { + "rewardFiles": ["reward.json", "reward.txt"], + "passingReward": 1 + } +} diff --git a/evals/external/terminal-bench/verifier.mjs b/evals/external/terminal-bench/verifier.mjs new file mode 100644 index 00000000..b37a7357 --- /dev/null +++ b/evals/external/terminal-bench/verifier.mjs @@ -0,0 +1,108 @@ +/** + * Terminal-Bench verifier artifact reading. + * + * Harbor's verifier writes a numeric reward to `logs/verifier/reward.json` + * (preferred) or `reward.txt` inside the trial's artifact tree. This module + * reads that evidence without re-implementing the verifier: parse the reward, + * grade it against the lock's passing threshold, pull pytest assertion counts + * when a test log is present, and hash the artifact tree so a trial's end + * state is auditable byte-for-byte. + */ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +/** Parse a reward artifact. Returns { reward, metrics } or null when unusable. */ +export function parseReward(content, filename) { + if (filename.endsWith('.json')) { + let metrics; + try { + metrics = JSON.parse(content); + } catch { + return null; + } + if (!metrics || typeof metrics !== 'object') return null; + if (typeof metrics.reward === 'number' && Number.isFinite(metrics.reward)) return { reward: metrics.reward, metrics }; + const numeric = Object.values(metrics).filter((v) => typeof v === 'number' && Number.isFinite(v)); + // A single numeric metric is unambiguous; anything else needs a human. + return { reward: numeric.length === 1 ? numeric[0] : null, metrics }; + } + const value = Number.parseFloat(String(content).trim()); + if (!Number.isFinite(value)) return null; + return { reward: value, metrics: { reward: value } }; +} + +export function verdictFromReward(reward, { passingReward = 1 } = {}) { + return typeof reward === 'number' && reward >= passingReward ? 'pass' : 'fail'; +} + +/** Extract assertion counts from a pytest summary line, if one exists. */ +export function parsePytestSummary(text) { + const passed = /(\d+) passed/.exec(text); + const failed = /(\d+) failed/.exec(text); + if (!passed && !failed) return null; + return { passed: passed ? Number(passed[1]) : 0, failed: failed ? Number(failed[1]) : 0 }; +} + +function walkFiles(dir) { + const out = []; + const stack = [dir]; + while (stack.length) { + const current = stack.pop(); + let entries; + try { + entries = fs.readdirSync(current, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + const full = path.join(current, entry.name); + if (entry.isDirectory()) stack.push(full); + else if (entry.isFile()) out.push(full); + } + } + return out.sort(); +} + +/** sha256 over sorted relative paths + contents: the trial's end-state fingerprint. */ +export function hashTree(dir) { + const hash = crypto.createHash('sha256'); + for (const file of walkFiles(dir)) { + hash.update(path.relative(dir, file)); + hash.update('\0'); + hash.update(fs.readFileSync(file)); + hash.update('\0'); + } + return hash.digest('hex'); +} + +/** + * Read a trial directory's verifier evidence: reward (reward.json preferred), + * pytest assertion counts from any *.log/*.txt test output, and the artifact + * tree hash. A missing reward stays null — it is never coerced to 0. + */ +export function collectVerifierEvidence(trialDir) { + const files = walkFiles(trialDir); + const rewardJson = files.find((f) => path.basename(f) === 'reward.json'); + const rewardTxt = files.find((f) => path.basename(f) === 'reward.txt'); + let reward = null; + let rewardPath = null; + let metrics = null; + for (const candidate of [rewardJson, rewardTxt]) { + if (!candidate) continue; + const parsed = parseReward(fs.readFileSync(candidate, 'utf8'), path.basename(candidate)); + if (parsed) { + reward = parsed.reward; + metrics = parsed.metrics; + rewardPath = candidate; + break; + } + } + let pytest = null; + for (const file of files) { + if (!/\.(log|txt|out)$/.test(file) || file === rewardTxt) continue; + pytest = parsePytestSummary(fs.readFileSync(file, 'utf8')); + if (pytest) break; + } + return { reward, rewardPath, metrics, pytest, treeHash: hashTree(trialDir) }; +} diff --git a/packages/harness/test/eval-tb-adapter.test.mjs b/packages/harness/test/eval-tb-adapter.test.mjs new file mode 100644 index 00000000..e295d357 --- /dev/null +++ b/packages/harness/test/eval-tb-adapter.test.mjs @@ -0,0 +1,134 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { + validateTaskLock, + verifyTaskAgainstLock, + stampTaskLock, + buildHarborRunArgs, + runHarbor, + findLatestJobDir, + readTrialResult, + classifyFailure, +} from '../../../evals/external/terminal-bench/harbor-adapter.mjs'; +import { hashTree } from '../../../evals/external/terminal-bench/verifier.mjs'; + +const LOCK = JSON.parse(fs.readFileSync(new URL('../../../evals/external/terminal-bench/task-lock.json', import.meta.url), 'utf8')); + +function tmpdir() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'tb-adapter-')); +} + +test('the committed task-lock.json is structurally valid and pins the plan task', () => { + const verdict = validateTaskLock(LOCK); + assert.deepEqual(verdict.errors, []); + assert.equal(verdict.ok, true); + assert.equal(LOCK.datasetRef, 'terminal-bench@2.0'); + assert.equal(LOCK.task, 'cobol-modernization'); +}); + +test('validateTaskLock names every missing field', () => { + const verdict = validateTaskLock({ task: 'x' }); + assert.equal(verdict.ok, false); + assert.ok(verdict.errors.some((e) => /datasetRef/.test(e))); + assert.ok(verdict.errors.some((e) => /verifier/.test(e))); +}); + +test('an unstamped lock fails task verification closed', () => { + const dir = tmpdir(); + fs.writeFileSync(path.join(dir, 'task.yaml'), 'name: cobol-modernization'); + const verdict = verifyTaskAgainstLock(dir, { ...LOCK, taskChecksum: null }); + assert.equal(verdict.ok, false); + assert.match(verdict.reason, /not.*stamped|unpinned|no checksum/i); +}); + +test('stampTaskLock pins the task directory and verification then passes and detects tampering', () => { + const dir = tmpdir(); + fs.writeFileSync(path.join(dir, 'task.yaml'), 'name: cobol-modernization'); + fs.mkdirSync(path.join(dir, 'tests')); + fs.writeFileSync(path.join(dir, 'tests', 'test_output.py'), 'def test(): pass'); + const stamped = stampTaskLock(dir, LOCK); + assert.equal(stamped.taskChecksum, hashTree(dir)); + assert.equal(verifyTaskAgainstLock(dir, stamped).ok, true); + fs.writeFileSync(path.join(dir, 'tests', 'test_output.py'), 'def test(): assert False'); + const tampered = verifyTaskAgainstLock(dir, stamped); + assert.equal(tampered.ok, false); + assert.match(tampered.reason, /checksum/i); +}); + +test('buildHarborRunArgs pins dataset, task, agent, model, and environment', () => { + const args = buildHarborRunArgs({ + lock: LOCK, + agentRef: 'evals.external.terminal_bench.harbor_agent:StdioBridgeAgent', + model: 'moonshotai/kimi-k2.7-code', + envName: 'daytona', + }); + assert.deepEqual(args, [ + 'run', + '-d', + 'terminal-bench@2.0', + '--task-name', + 'cobol-modernization', + '--agent', + 'evals.external.terminal_bench.harbor_agent:StdioBridgeAgent', + '--model', + 'moonshotai/kimi-k2.7-code', + '--env', + 'daytona', + '-n', + '1', + ]); +}); + +test('runHarbor uses the injected spawn and surfaces exit details', () => { + let seen = null; + const spawnImpl = (cmd, args, opts) => { + seen = { cmd, args, opts }; + return { status: 0, stdout: 'done', stderr: '' }; + }; + const result = runHarbor({ args: ['run', '-d', 'x'], cwd: '/work', spawnImpl, timeoutMs: 1000 }); + assert.equal(seen.cmd, 'harbor'); + assert.deepEqual(seen.args, ['run', '-d', 'x']); + assert.equal(seen.opts.cwd, '/work'); + assert.equal(seen.opts.timeout, 1000); + assert.equal(result.code, 0); + assert.equal(result.timedOut, false); + assert.equal(result.spawnError, null); +}); + +test('runHarbor classifies a missing harbor binary and a timeout', () => { + const enoent = runHarbor({ args: [], cwd: '.', spawnImpl: () => ({ status: null, error: Object.assign(new Error('nf'), { code: 'ENOENT' }) }) }); + assert.equal(enoent.spawnError, 'ENOENT'); + const timeout = runHarbor({ args: [], cwd: '.', spawnImpl: () => ({ status: null, error: Object.assign(new Error('t'), { code: 'ETIMEDOUT' }) }) }); + assert.equal(timeout.timedOut, true); +}); + +test('findLatestJobDir picks the newest job directory', () => { + const root = tmpdir(); + fs.mkdirSync(path.join(root, '2026-07-30__10-00-00')); + fs.mkdirSync(path.join(root, '2026-07-30__11-00-00')); + fs.writeFileSync(path.join(root, 'not-a-dir.txt'), 'x'); + assert.equal(findLatestJobDir(root), path.join(root, '2026-07-30__11-00-00')); + assert.equal(findLatestJobDir(path.join(root, 'missing')), null); +}); + +test('readTrialResult finds verifier evidence inside the job tree', () => { + const job = tmpdir(); + const verifierDir = path.join(job, 'trial-0', 'artifacts', 'logs', 'verifier'); + fs.mkdirSync(verifierDir, { recursive: true }); + fs.writeFileSync(path.join(verifierDir, 'reward.json'), '{"reward": 1}'); + const result = readTrialResult(job); + assert.equal(result.reward, 1); + assert.equal(result.verdict, 'pass'); +}); + +test('classifyFailure distinguishes infrastructure, provider, verifier, and valid trials', () => { + assert.equal(classifyFailure({ run: { spawnError: 'ENOENT', code: null, timedOut: false }, reward: null }), 'infrastructure'); + assert.equal(classifyFailure({ run: { spawnError: null, code: null, timedOut: true }, reward: null }), 'infrastructure'); + assert.equal(classifyFailure({ run: { spawnError: null, code: 0, timedOut: false }, reward: null, providerFailure: true }), 'provider'); + assert.equal(classifyFailure({ run: { spawnError: null, code: 0, timedOut: false }, reward: null }), 'verifier'); + assert.equal(classifyFailure({ run: { spawnError: null, code: 0, timedOut: false }, reward: 1 }), null); + assert.equal(classifyFailure({ run: { spawnError: null, code: 0, timedOut: false }, reward: 0 }), null, 'reward 0 is a graded fail, not an infrastructure failure'); +}); diff --git a/packages/harness/test/eval-tb-agent.test.mjs b/packages/harness/test/eval-tb-agent.test.mjs new file mode 100644 index 00000000..32721282 --- /dev/null +++ b/packages/harness/test/eval-tb-agent.test.mjs @@ -0,0 +1,123 @@ +import assert from 'node:assert/strict'; +import { PassThrough } from 'node:stream'; +import { test } from 'node:test'; +import { BRIDGE_TOOLS, runStdioAgent } from '../../../evals/external/terminal-bench/agent.mjs'; +import { replayDriver, ProviderError } from '../../../evals/lib/drivers.mjs'; +import { createTelemetry } from '../../../evals/lib/telemetry.mjs'; + +/** + * Simulated Harbor side of the protocol: answers every exec line with a + * scripted result and collects everything the agent writes. + */ +function pump({ resultFor = () => ({ code: 0, stdout: 'ok', stderr: '' }) } = {}) { + const input = new PassThrough(); + const output = new PassThrough(); + const lines = []; + let buffer = ''; + output.on('data', (chunk) => { + buffer += chunk.toString(); + let idx; + while ((idx = buffer.indexOf('\n')) >= 0) { + const line = JSON.parse(buffer.slice(0, idx)); + lines.push(line); + buffer = buffer.slice(idx + 1); + if (line.type === 'exec') { + input.write(`${JSON.stringify({ type: 'result', id: line.id, ...resultFor(line) })}\n`); + } + } + }); + return { input, output, lines }; +} + +test('bridge tools expose exactly a terminal and a finish', () => { + assert.deepEqual( + BRIDGE_TOOLS.map((t) => t.name).sort(), + ['bash', 'finish'] + ); +}); + +test('happy path: execs stream out, results stream back into the driver, done carries the answer', async () => { + const observed = []; + const driver = { + next: (() => { + const actions = [ + { type: 'tool', name: 'bash', input: { command: 'ls' } }, + { type: 'tool', name: 'bash', input: { command: 'cat main.cobol' } }, + { type: 'finish', answer: 'reimplemented', stopReason: 'model_finish' }, + ]; + let i = 0; + return async () => actions[i++]; + })(), + observe: (action, result) => observed.push({ action: action.input.command, result }), + }; + const { input, output, lines } = pump({ resultFor: (line) => ({ code: 0, stdout: `ran:${JSON.parse('{}') ? line.command : ''}`, stderr: '' }) }); + const done = await runStdioAgent({ driver, input, output, systemPrompt: 's', instruction: 'i' }); + assert.equal(done.stopReason, 'model_finish'); + assert.equal(done.answer, 'reimplemented'); + const execs = lines.filter((l) => l.type === 'exec'); + assert.deepEqual( + execs.map((e) => e.command), + ['ls', 'cat main.cobol'] + ); + assert.equal(observed.length, 2, 'every exec result is observed by the driver'); + assert.equal(lines.at(-1).type, 'done'); +}); + +test('a provider failure surfaces as provider_error with its classification', async () => { + const driver = { + next: async () => { + throw new ProviderError('boom', { kind: 'network', billed: false }); + }, + }; + const { input, output, lines } = pump(); + const done = await runStdioAgent({ driver, input, output, systemPrompt: 's', instruction: 'i' }); + assert.equal(done.stopReason, 'provider_error'); + assert.equal(done.providerFailure.kind, 'network'); + assert.equal(done.providerFailure.billed, false); + assert.equal(lines.at(-1).type, 'done'); +}); + +test('the step ceiling ends the run with max_steps', async () => { + const driver = { next: async () => ({ type: 'tool', name: 'bash', input: { command: 'true' } }) }; + const { input, output } = pump(); + const done = await runStdioAgent({ driver, input, output, maxSteps: 3, systemPrompt: 's', instruction: 'i' }); + assert.equal(done.stopReason, 'max_steps'); + assert.equal(done.steps, 3); +}); + +test('a budget-exhausted finish passes its stop reason through', async () => { + const driver = replayDriver([{ type: 'finish', answer: '', stopReason: 'budget_exhausted' }]); + const { input, output } = pump(); + const done = await runStdioAgent({ driver, input, output, systemPrompt: 's', instruction: 'i' }); + assert.equal(done.stopReason, 'budget_exhausted'); +}); + +test('telemetry snapshot rides along in the done message', async () => { + const telemetry = createTelemetry(); + telemetry.record('request', { model: 'kimi' }); + const driver = replayDriver([{ type: 'finish', answer: 'x', stopReason: 'model_finish' }]); + const { input, output, lines } = pump(); + await runStdioAgent({ driver, input, output, telemetry, systemPrompt: 's', instruction: 'i' }); + const done = lines.at(-1); + assert.equal(done.telemetry.events[0].model, 'kimi'); +}); + +test('a malformed result line ends the run as a protocol_error', async () => { + const driver = { next: async () => ({ type: 'tool', name: 'bash', input: { command: 'ls' } }) }; + const input = new PassThrough(); + const output = new PassThrough(); + const lines = []; + let buffer = ''; + output.on('data', (chunk) => { + buffer += chunk.toString(); + let idx; + while ((idx = buffer.indexOf('\n')) >= 0) { + const line = JSON.parse(buffer.slice(0, idx)); + lines.push(line); + buffer = buffer.slice(idx + 1); + if (line.type === 'exec') input.write('this is not json\n'); + } + }); + const done = await runStdioAgent({ driver, input, output, systemPrompt: 's', instruction: 'i' }); + assert.equal(done.stopReason, 'protocol_error'); +}); diff --git a/packages/harness/test/eval-tb-conditions.test.mjs b/packages/harness/test/eval-tb-conditions.test.mjs new file mode 100644 index 00000000..df64ac57 --- /dev/null +++ b/packages/harness/test/eval-tb-conditions.test.mjs @@ -0,0 +1,46 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { buildGenericCondition, NEUTRAL_SYSTEM_PROMPT } from '../../../evals/external/terminal-bench/generic-condition.mjs'; +import { buildHarnessCondition } from '../../../evals/external/terminal-bench/harness-condition.mjs'; + +const INSTRUCTION = 'Reimplement the COBOL program in Python producing identical output files.'; +const LIMITS = { maxSteps: 60, timeoutMs: 15 * 60_000, maxOutputTokens: 8192, trialCeilingUsd: 5 }; +const CONTRACT = '# Engineer Agent contract\nOrient before edits; gate mutations; verify before completion.'; + +test('generic condition keeps the original instruction and a neutral prompt with no harness workflow', () => { + const condition = buildGenericCondition({ instruction: INSTRUCTION, limits: LIMITS }); + assert.equal(condition.id, 'generic'); + assert.equal(condition.instruction, INSTRUCTION); + assert.equal(condition.systemPrompt, NEUTRAL_SYSTEM_PROMPT); + assert.ok(!/harness|orient|gate|plan[_ -]?lock|skill/i.test(condition.systemPrompt), 'neutral prompt must not leak harness workflow'); + assert.deepEqual(condition.setupCommands, []); +}); + +test('the neutral prompt is a fair baseline that still encourages testing and verification', () => { + assert.match(NEUTRAL_SYSTEM_PROMPT, /verif|test/i); +}); + +test('harness condition layers the engineer contract and guidance on the same baseline', () => { + const condition = buildHarnessCondition({ instruction: INSTRUCTION, limits: LIMITS, engineerContract: CONTRACT, guidance: '## Skill: ensure-plan' }); + assert.equal(condition.id, 'harness'); + assert.equal(condition.instruction, INSTRUCTION, 'instruction must be byte-identical across conditions'); + assert.ok(condition.systemPrompt.startsWith(NEUTRAL_SYSTEM_PROMPT), 'treatment starts from the same neutral baseline'); + assert.ok(condition.systemPrompt.includes(CONTRACT)); + assert.ok(condition.systemPrompt.includes('## Skill: ensure-plan')); + assert.ok(condition.setupCommands.length > 0 && condition.setupCommands.every((c) => /harness/.test(c)), 'activation commands run the harness CLI'); +}); + +test('both conditions receive identical, independent limit copies', () => { + const generic = buildGenericCondition({ instruction: INSTRUCTION, limits: LIMITS }); + const harness = buildHarnessCondition({ instruction: INSTRUCTION, limits: LIMITS, engineerContract: CONTRACT }); + assert.deepEqual(generic.limits, harness.limits); + generic.limits.maxSteps = 1; + assert.equal(harness.limits.maxSteps, 60, 'mutating one condition must not leak into the other'); + assert.equal(LIMITS.maxSteps, 60, 'the shared input must not be mutated'); +}); + +test('condition builders reject missing instruction or limits', () => { + assert.throws(() => buildGenericCondition({ limits: LIMITS }), /instruction/); + assert.throws(() => buildGenericCondition({ instruction: INSTRUCTION }), /limits/); + assert.throws(() => buildHarnessCondition({ instruction: INSTRUCTION, limits: LIMITS }), /engineerContract/); +}); diff --git a/packages/harness/test/eval-tb-verifier.test.mjs b/packages/harness/test/eval-tb-verifier.test.mjs new file mode 100644 index 00000000..9049abe9 --- /dev/null +++ b/packages/harness/test/eval-tb-verifier.test.mjs @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { + parseReward, + verdictFromReward, + parsePytestSummary, + hashTree, + collectVerifierEvidence, +} from '../../../evals/external/terminal-bench/verifier.mjs'; + +function tmpdir() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'tb-verifier-')); +} + +test('parseReward reads harbor reward.json with a reward key', () => { + assert.deepEqual(parseReward('{"reward": 1}', 'reward.json'), { reward: 1, metrics: { reward: 1 } }); + assert.deepEqual(parseReward('{"reward": 0.25}', 'reward.json'), { reward: 0.25, metrics: { reward: 0.25 } }); +}); + +test('parseReward falls back to a single numeric metric when no reward key exists', () => { + assert.deepEqual(parseReward('{"accuracy": 0.5}', 'reward.json'), { reward: 0.5, metrics: { accuracy: 0.5 } }); + const ambiguous = parseReward('{"a": 1, "b": 0}', 'reward.json'); + assert.equal(ambiguous.reward, null, 'two metrics with no reward key is ambiguous'); +}); + +test('parseReward reads reward.txt plain numbers and rejects garbage', () => { + assert.equal(parseReward('1\n', 'reward.txt').reward, 1); + assert.equal(parseReward('0', 'reward.txt').reward, 0); + assert.equal(parseReward('not-a-number', 'reward.txt'), null); + assert.equal(parseReward('{invalid json', 'reward.json'), null); +}); + +test('verdictFromReward compares against the passing reward', () => { + assert.equal(verdictFromReward(1), 'pass'); + assert.equal(verdictFromReward(0.99), 'fail'); + assert.equal(verdictFromReward(0.5, { passingReward: 0.5 }), 'pass'); + assert.equal(verdictFromReward(null), 'fail'); +}); + +test('parsePytestSummary extracts passed and failed counts', () => { + assert.deepEqual(parsePytestSummary('==== 3 passed, 1 failed in 0.52s ===='), { passed: 3, failed: 1 }); + assert.deepEqual(parsePytestSummary('5 passed in 1.2s'), { passed: 5, failed: 0 }); + assert.deepEqual(parsePytestSummary('2 failed in 0.1s'), { passed: 0, failed: 2 }); + assert.equal(parsePytestSummary('no tests ran'), null); +}); + +test('hashTree is deterministic, content-sensitive, and path-sensitive', () => { + const a = tmpdir(); + const b = tmpdir(); + for (const dir of [a, b]) { + fs.mkdirSync(path.join(dir, 'sub'), { recursive: true }); + fs.writeFileSync(path.join(dir, 'sub', 'x.txt'), 'hello'); + fs.writeFileSync(path.join(dir, 'y.txt'), 'world'); + } + assert.equal(hashTree(a), hashTree(b), 'identical trees hash identically'); + fs.writeFileSync(path.join(b, 'y.txt'), 'world!'); + assert.notEqual(hashTree(a), hashTree(b), 'content change changes the hash'); + fs.writeFileSync(path.join(b, 'y.txt'), 'world'); + fs.renameSync(path.join(b, 'y.txt'), path.join(b, 'z.txt')); + assert.notEqual(hashTree(a), hashTree(b), 'path change changes the hash'); +}); + +test('collectVerifierEvidence prefers reward.json, captures pytest counts, and hashes the tree', () => { + const trial = tmpdir(); + const verifierDir = path.join(trial, 'artifacts', 'logs', 'verifier'); + fs.mkdirSync(verifierDir, { recursive: true }); + fs.writeFileSync(path.join(verifierDir, 'reward.txt'), '0'); + fs.writeFileSync(path.join(verifierDir, 'reward.json'), '{"reward": 1}'); + fs.writeFileSync(path.join(verifierDir, 'pytest.log'), '==== 4 passed, 2 failed in 1.0s ===='); + const evidence = collectVerifierEvidence(trial); + assert.equal(evidence.reward, 1, 'reward.json wins over reward.txt'); + assert.match(evidence.rewardPath, /reward\.json$/); + assert.deepEqual(evidence.pytest, { passed: 4, failed: 2 }); + assert.match(evidence.treeHash, /^[0-9a-f]{64}$/); +}); + +test('collectVerifierEvidence reports a missing reward as null evidence, not zero', () => { + const trial = tmpdir(); + fs.mkdirSync(path.join(trial, 'artifacts'), { recursive: true }); + const evidence = collectVerifierEvidence(trial); + assert.equal(evidence.reward, null); + assert.equal(evidence.rewardPath, null); +});