diff --git a/.gitignore b/.gitignore index 993c3ad37..8efc2430d 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,6 @@ src/deadline/client/ui/_translation_keys.py # Install builder license license.xml /THIRD_PARTY_LICENSES + +# Agent evals run artifacts +/evals/output/ diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 000000000..e1f191018 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,149 @@ +# Agent evals + +Measure how well an AI agent achieves goals with Deadline Cloud tooling and +docs — and prove, with a before-and-after comparison, that a change to the CLI, +the docs, or any reference material actually helps. + +## How it works + +1. An **eval file** (JSON) gives an agent a goal, the tools it may use, optional + reference material, and a **rubric** describing what a passing answer must do. +2. The **runner** launches an isolated headless agent (`claude -p`) per run in a + throwaway sandbox and captures telemetry (tool calls, turns, cost). +3. An **LLM judge** grades the agent's final answer against the rubric. +4. In **A/B mode**, every case runs twice — with this repo at the current ref + (baseline) and at `--revised-ref` (candidate) — and the summary reports the + paired delta plus the diff, which is the PR-ready proposed change. + +The material under test can be almost anything the agent relies on: + +| Data source | How | +| --- | --- | +| `deadline` CLI | `pip install -e .` this repo; A/B two git refs of `src/` | +| AWS CLI usage | goal + rubric only — no subject needed | +| AWS documentation / blog / web page | fetch it to markdown, pass as `materials`, seed a corpus to revise | +| This repo's docs | A/B with `--pathspec ':(glob)docs/**/*.md' --seed-subject` | +| Real Deadline Cloud (submit a job) | `"env": "real_aws"` case + `--allow-real-aws` and sandbox env vars (see below) | + +## Setup + +```bash +pip install -e . # the agent's `deadline` is this checkout (A/B needs this) +which claude # Claude Code must be on PATH and authenticated +``` + +That's it — no extra dependencies; the evals use only the Python standard library. + +## Run + +```bash +cd evals + +# score how an agent does today (baseline only) +python -m agent_evals.runner run examples/deadline_cli.json --k 3 + +# A/B a change: current branch vs a revision of the CLI source +python -m agent_evals.runner run examples/deadline_cli.json --k 3 --revised-ref my-improvement + +# A/B a docs change instead of code +# --seed-subject copies the owned files into each run's sandbox (under subject/) so +# the agent actually reads them; without it a docs A/B compares identical sandboxes. +python -m agent_evals.runner run my_docs_eval.json --revised-ref docs-fix \ + --pathspec ':(glob)docs/**/*.md' --seed-subject +``` + +Flags: + +| Flag | Effect | +| --- | --- | +| `--k N` | runs per case per variant (default 1); a case's own `k` overrides it | +| `--model ALIAS` | model for both the agent and the judge | +| `--revised-ref REF` | enable A/B mode: also run with the repo at this git ref | +| `--base-ref REF` | baseline ref for A/B (default: current branch) | +| `--pathspec SPEC` | repo paths the A/B owns, e.g. `src` or `:(glob)docs/**/*.md` | +| `--seed-subject` | copy the subject's owned files into each sandbox (docs A/B — see above) | +| `--allow-real-aws` | opt in to `real_aws` cases, which submit real jobs (see below) | + +Artifacts land under `evals/output///`: per-run telemetry and +transcripts, `summary.json`, and — when a revision measurably improved an eval — +`proposal.patch`. Exit code 4 means the revision regressed. + +## Real-AWS evals (submit real jobs) + +A case with `"env": "real_aws"` (see `examples/real_aws_submit.json`) has the agent +submit a real job bundle to a real farm and confirm it reaches SUCCEEDED — a true +end-to-end check. Because these submit **real, billable jobs**, they are opt-in and +never name an account in the eval file: + +- They are **skipped** (not failed) unless you pass `--allow-real-aws`, so a plain + `run` stays green in CI. +- The farm/queue come from environment variables — set them to a **non-production + sandbox you own**: + +```bash +export DEADLINE_EVAL_FARM_ID=farm-... +export DEADLINE_EVAL_QUEUE_ID=queue-... +export DEADLINE_EVAL_REGION=us-west-2 # optional +deadline auth login # the runner prechecks auth +python -m agent_evals.runner run examples/real_aws_submit.json --allow-real-aws +``` + +The case's `prompt`/`rubric` may reference `{farm_id}`, `{queue_id}`, and `{region}`, +which are filled from those env vars. Missing vars or expired auth skip the case +with a clear reason rather than submitting to the wrong place. + +## Write an eval + +```json +[ + { + "id": "my_case", + "prompt": "The goal, stated imperatively and self-contained.", + "tools": ["Bash", "Read"], + "rubric": "What a passing answer must do, in plain language.", + "materials": {"guide.md": "optional reference text the agent can read"}, + "max_turns": 20, + "k": 3, + "env": "real_aws" + } +] +``` + +Fields: `id`, `prompt`, and `rubric` are required; the rest are optional. + +| Field | Meaning | +| --- | --- | +| `tools` | Claude Code tools the agent may use (default: `Bash`, `Read`, `Write`, `Edit`) | +| `materials` | `{path: content}` written under `materials/` in the sandbox and named in the prompt; keys may contain `/` but not `..` or absolute paths | +| `max_turns` | agent turn cap (default 20) | +| `k` | runs per variant for this case (overrides the `--k` flag) | +| `env` | set to `real_aws` to mark a case that submits real jobs — see above; omit for ordinary offline cases | + +The rubric is the only per-eval authoring step that matters: it should state the +*material's own* success criterion (for a docs page, what the page promises the +reader can do), including what a correct answer looks like when the evidence is +incomplete — a good judge passes an agent that refuses to invent missing details. +For `real_aws` cases, `prompt` and `rubric` may use `{farm_id}` / `{queue_id}` / +`{region}` placeholders, filled from the environment variables above. + +## Close the loop automatically + +`reviser.revise()` hands a struggling run's transcript to an agent that edits the +subject (code or docs), commits to a scratch ref, and returns it — feed that ref +back to `--revised-ref` to A/B-prove the improvement: + +```python +from agent_evals import reviser, subject + +subj = subject.repo_subject() # or corpus_subject(markdown) +ref = reviser.revise(subj, run_dir, goal="...", base_ref="mainline") +# python -m agent_evals.runner run my_eval.json --revised-ref +``` + +## Notes + +- The tested agent always runs isolated — the orchestrating session must never do + the goal itself, or the telemetry measures the wrong thing. +- Runs that talk to real AWS use whatever credentials/config the environment has; + point them at a non-production sandbox account. +- The judge is a single vote per run; for gate-quality decisions increase `k`. diff --git a/evals/agent_evals/__init__.py b/evals/agent_evals/__init__.py new file mode 100644 index 000000000..f245fa943 --- /dev/null +++ b/evals/agent_evals/__init__.py @@ -0,0 +1,4 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Agent evals: measure how well an AI agent achieves goals with Deadline Cloud +tooling and docs, and A/B-prove improvements to whatever the agent relied on.""" diff --git a/evals/agent_evals/harness.py b/evals/agent_evals/harness.py new file mode 100644 index 000000000..61177d4d6 --- /dev/null +++ b/evals/agent_evals/harness.py @@ -0,0 +1,155 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Run Claude Code headless against a goal and capture telemetry. + +The tested agent runs as an ISOLATED subprocess (`claude -p` with stream-json +output) in a sandbox directory. The JSON event stream carries both per-tool-call +events and a final result event with token/cost telemetry, so no extra +instrumentation is needed. +""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +# Wall-clock ceiling for a single agent run. --max-turns bounds model turns, but a +# hung network/auth interaction has no turn cost, so without this one stuck run +# could block the whole batch indefinitely. +DEFAULT_TIMEOUT_S = 1200 + + +class HarnessError(RuntimeError): + """Raised when the agent subprocess cannot be launched or times out.""" + + +@dataclass +class RunResult: + """One headless agent run: outcome + telemetry, plus the raw event log.""" + + success: bool # did the CLI complete without error + subtype: Optional[str] # result subtype, e.g. "success" / "error_max_turns" + tool_calls: list = field(default_factory=list) + num_turns: int = 0 + total_cost_usd: float = 0.0 + duration_ms: int = 0 + final_text: str = "" # the agent's final response text + workdir: Optional[Path] = None + raw_events: list = field(default_factory=list) + + @property + def tool_call_count(self) -> int: + return len(self.tool_calls) + + def telemetry_dict(self) -> dict: + """Serializable telemetry (excludes the bulky raw event log).""" + return { + "success": self.success, + "subtype": self.subtype, + "tool_calls": self.tool_calls, + "tool_call_count": self.tool_call_count, + "num_turns": self.num_turns, + "total_cost_usd": self.total_cost_usd, + "duration_ms": self.duration_ms, + "final_text": self.final_text, + } + + +def run_agent( + prompt: str, + workdir: Path, + allowed_tools: list, + *, + max_turns: int = 20, + model: Optional[str] = None, + claude_bin: str = "claude", + timeout_s: int = DEFAULT_TIMEOUT_S, +) -> RunResult: + """Run Claude Code headless in `workdir`, restricted to `allowed_tools`. + + Raises HarnessError if the agent binary can't be launched or the run exceeds + `timeout_s`, so one bad run surfaces a clear error rather than aborting the + batch with a raw traceback or hanging forever. + """ + cmd = [ + claude_bin, + "-p", + prompt, + "--output-format", + "stream-json", + "--verbose", # required for stream-json event detail + "--permission-mode", + "bypassPermissions", + "--max-turns", + str(max_turns), + ] + if allowed_tools: + cmd += ["--allowedTools", *allowed_tools] + if model: + cmd += ["--model", model] + + # stdin=DEVNULL: with -p the CLI still waits on stdin and can exit non-zero on a + # closed pipe; DEVNULL makes the call cleanly non-interactive. + try: + proc = subprocess.run( + cmd, + cwd=str(workdir), + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + timeout=timeout_s, + ) + except OSError as e: + raise HarnessError(f"could not launch {claude_bin}: {e}") from e + except subprocess.TimeoutExpired as e: + raise HarnessError(f"agent run exceeded {timeout_s}s wall-clock timeout") from e + + events = [] + for line in proc.stdout.splitlines(): + line = line.strip() + if not line: + continue + try: + events.append(json.loads(line)) + except json.JSONDecodeError: + continue # non-JSON lines (rare) are ignored + + return _parse_events(events, proc.returncode, workdir) + + +def _parse_events(events: list, returncode: int, workdir: Path) -> RunResult: + tool_calls = [] + final = None + + for ev in events: + if ev.get("type") == "assistant": + for blk in ev.get("message", {}).get("content", []): + if blk.get("type") == "tool_use": + tool_calls.append(blk["name"]) + elif ev.get("type") == "result": + final = ev + + if final is None: + # CLI died before emitting a result event. + return RunResult( + success=False, + subtype="no_result_event", + tool_calls=tool_calls, + workdir=workdir, + raw_events=events, + ) + + return RunResult( + success=(not final.get("is_error", False)) and returncode == 0, + subtype=final.get("subtype"), + tool_calls=tool_calls, + num_turns=final.get("num_turns", 0), + total_cost_usd=final.get("total_cost_usd", 0.0), + duration_ms=final.get("duration_ms", 0), + final_text=final.get("result", "") if isinstance(final.get("result"), str) else "", + workdir=workdir, + raw_events=events, + ) diff --git a/evals/agent_evals/judge.py b/evals/agent_evals/judge.py new file mode 100644 index 000000000..928cab044 --- /dev/null +++ b/evals/agent_evals/judge.py @@ -0,0 +1,176 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Grade an agent's answer against a natural-language rubric (LLM-as-judge). + +The judge is a SCORER, not an agent: a headless, tool-less, single-turn model call. +It sees only the rubric, the task prompt, and the agent's final answer, so its +verdict is reproducible from the run artifact alone. The rubric being a plain +string is what makes new eval domains code-free: describing what "success" means +is the only per-eval authoring step. +""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import dataclass +from typing import Optional + + +# Wall-clock ceiling for one judge call. --max-turns bounds model turns, but a hung +# network/auth interaction has no turn cost, and the judge runs once per run in the +# batch -- so without this a single stuck judge would hang the whole eval. +DEFAULT_TIMEOUT_S = 300 + + +class JudgeError(RuntimeError): + """Raised when the judge call fails to produce a usable verdict.""" + + +@dataclass +class Verdict: + passed: bool + reasoning: str + + +_JUDGE_PROMPT = """\ +You are grading an AI agent's answer to a task. Decide ONLY whether the answer +satisfies the rubric below. Judge the substance of the answer, not its wording or +format. A correct conclusion stated in unexpected phrasing still passes; a +confident answer that is wrong or unsupported fails. If the rubric asks the agent +to reach a conclusion the available evidence does not support, then correctly +declining to invent one SATISFIES the rubric. + +You have NO tools. Do not attempt to read any file or run any command -- files the +answer mentions are not available to you. Grade purely from the text below. + +=== RUBRIC (what a passing answer must do) === +{rubric} + +=== THE TASK THE AGENT WAS GIVEN === +{prompt} + +=== THE AGENT'S FINAL ANSWER === +{answer} + +Respond with ONLY a JSON object on a single line, no prose, no code fence: +{{"passed": true or false, "reasoning": "one or two sentences citing the rubric"}} +""" + + +def judge_answer( + rubric: str, + prompt: str, + answer: str, + *, + model: Optional[str] = None, + claude_bin: str = "claude", + timeout_s: int = DEFAULT_TIMEOUT_S, +) -> Verdict: + """Grade `answer` against `rubric` with a headless, tool-less model call. + + Raises JudgeError if the judge binary can't be launched, the call exceeds + `timeout_s`, or the reply carries no usable verdict. + """ + if not (answer or "").strip(): + return Verdict(passed=False, reasoning="agent produced no final answer") + + full_prompt = _JUDGE_PROMPT.format(rubric=rubric, prompt=prompt or "(none)", answer=answer) + cmd = [ + claude_bin, + "-p", + full_prompt, + "--output-format", + "json", + "--permission-mode", + "bypassPermissions", + # The judge must answer from the given text alone, so all tools are denied. + # A denied tool attempt still consumes a turn, so leave headroom for the + # model to recover and answer instead of dying on error_max_turns. + "--max-turns", + "5", + "--disallowedTools", + "Bash", + "Read", + "Write", + "Edit", + "Grep", + "Glob", + "WebFetch", + "WebSearch", + "Agent", + "TodoWrite", + "NotebookEdit", + ] + if model: + cmd += ["--model", model] + + try: + # stdin=DEVNULL: with -p the CLI still waits on stdin and can exit non-zero + # on a closed pipe. + proc = subprocess.run( + cmd, capture_output=True, text=True, stdin=subprocess.DEVNULL, timeout=timeout_s + ) + except OSError as e: + raise JudgeError(f"could not launch {claude_bin}: {e}") from e + except subprocess.TimeoutExpired as e: + raise JudgeError(f"judge call exceeded {timeout_s}s wall-clock timeout") from e + + # --output-format json wraps the reply in an envelope whose `result` field is + # the text we asked for. + reply = proc.stdout + try: + envelope = json.loads(proc.stdout) + if isinstance(envelope, dict) and isinstance(envelope.get("result"), str): + reply = envelope["result"] + except json.JSONDecodeError: + # stdout wasn't the JSON envelope (e.g. plain-text or truncated output); + # fall through and try to grade the raw stdout instead. + pass + + # Parse the verdict from stdout FIRST: the CLI sometimes exits non-zero after + # emitting a valid result (e.g. a model-availability warning). Only surface the + # exit code when stdout carried nothing gradable. + try: + return _extract_verdict(reply) + except JudgeError: + if proc.returncode != 0: + raise JudgeError( + f"judge exited {proc.returncode} with no usable verdict: " + f"{(proc.stderr or reply).strip()[:200]}" + ) from None + raise + + +def _extract_verdict(text: str) -> Verdict: + """Pull the JSON verdict out of the judge's reply, tolerating code fences or + stray prose around it (scan first '{' to last '}').""" + start = text.find("{") + end = text.rfind("}") + if start == -1 or end == -1 or end < start: + raise JudgeError(f"judge response had no JSON object: {text[:200]!r}") + try: + obj = json.loads(text[start : end + 1]) + except json.JSONDecodeError as e: + raise JudgeError(f"judge JSON did not parse: {e}") from e + if "passed" not in obj: + raise JudgeError(f"judge JSON missing 'passed': {obj!r}") + return Verdict(passed=_coerce_passed(obj["passed"]), reasoning=str(obj.get("reasoning", ""))) + + +def _coerce_passed(value: object) -> bool: + """Safely coerce the judge's 'passed' field to a Python bool. + + The prompt asks for a JSON boolean, but models sometimes emit strings. Plain + bool("false") is True in Python, which would silently flip a FAIL to a PASS -- + so we handle known string forms explicitly and reject anything ambiguous. + """ + if isinstance(value, bool): + return value + if isinstance(value, str): + low = value.strip().lower() + if low in ("true", "pass", "yes", "1"): + return True + if low in ("false", "fail", "no", "0"): + return False + raise JudgeError(f"judge 'passed' field has ambiguous value {value!r}; expected a JSON boolean") diff --git a/evals/agent_evals/reviser.py b/evals/agent_evals/reviser.py new file mode 100644 index 000000000..f1744f278 --- /dev/null +++ b/evals/agent_evals/reviser.py @@ -0,0 +1,128 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Propose improvements to a subject from a run where the agent struggled. + +Given a struggling run's transcript and a Subject (the repo source, repo docs, or +a seeded corpus), drive an isolated agent to edit the subject so the NEXT agent +succeeds more reliably, and commit the edits to a scratch git ref. Re-running the +eval with --revised-ref then A/B-proves whether the edit helped, and +the diff is the PR-ready proposal. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +from .subject import Subject + + +# Bounds for the reviser agent. It drives a full edit session, so it needs both a +# turn ceiling and a wall-clock ceiling -- otherwise a runaway or hung session has +# no limit at all. +DEFAULT_MAX_TURNS = 40 +DEFAULT_TIMEOUT_S = 1200 + + +class ReviseError(RuntimeError): + """Raised when the reviser cannot run or produces no change to the subject.""" + + +_REVISE_PROMPT = """\ +You are improving the material an AI agent relies on (source code, documentation, +or a guide) so the agent can achieve a goal more reliably. The material is in this +directory. Below is the goal and a transcript of an agent that used the CURRENT +material and struggled (took too many steps, went down wrong paths, or failed). + +Edit the material to fix what tripped the agent up: +- add a missing prerequisite, command, or step the agent had to guess at, +- clarify anything ambiguous the agent misread, +- if you change code, it MUST fully work end to end -- documenting behavior that + isn't implemented is worse than no change. + +Make additive, minimal edits; do not remove existing behavior or guidance. When +done, STOP -- do not run tests or git. + +=== THE GOAL THE AGENT WAS GIVEN === +{goal} + +=== AGENT TRANSCRIPT (struggled) === +{transcript} +""" + + +def transcript_text(events_path: Path, max_parts: int = 60) -> str: + """A compact transcript of assistant text + tool calls, for the revise prompt.""" + parts = [] + for line in events_path.read_text().splitlines(): + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue + if ev.get("type") != "assistant": + continue + for blk in ev.get("message", {}).get("content", []): + if blk.get("type") == "text": + parts.append("ASSISTANT: " + blk["text"][:500]) + elif blk.get("type") == "tool_use": + inp = blk.get("input", {}) + detail = inp.get("command") or inp.get("file_path") or json.dumps(inp)[:200] + parts.append(f"TOOL[{blk.get('name')}]: {str(detail)[:300]}") + return "\n".join(parts[:max_parts]) + + +def revise( + subj: Subject, + run_dir: Path, + *, + goal: str, + base_ref: str, + claude_bin: str = "claude", + max_turns: int = DEFAULT_MAX_TURNS, + timeout_s: int = DEFAULT_TIMEOUT_S, +) -> str: + """Edit the subject from a struggling run's transcript; commit to a scratch ref. + + Returns the scratch ref name. Raises ReviseError if the agent can't be launched, + exceeds `timeout_s`, or made no edits -- a candidate identical to baseline would + make any measured delta pure noise. + """ + prompt = _REVISE_PROMPT.format(goal=goal, transcript=transcript_text(run_dir / "events.jsonl")) + + subj.checkout(base_ref) # edit from a clean baseline + try: + proc = subprocess.run( + [ + claude_bin, + "-p", + prompt, + "--permission-mode", + "bypassPermissions", + "--max-turns", + str(max_turns), + "--allowedTools", + "Read", + "Edit", + "Write", + "Grep", + "Glob", + ], + cwd=str(subj.root), + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + timeout=timeout_s, + ) + except OSError as e: + raise ReviseError(f"could not launch {claude_bin}: {e}") from e + except subprocess.TimeoutExpired as e: + raise ReviseError(f"reviser agent exceeded {timeout_s}s wall-clock timeout") from e + if proc.returncode != 0: + raise ReviseError(f"reviser agent exited {proc.returncode}: {proc.stderr.strip()[:200]}") + + if not subj.capture_diff().strip(): + raise ReviseError("reviser agent made no edits to the subject.") + + ref = f"eval-revise-{run_dir.parent.parent.name}" + return subj.commit_scratch(ref, "agent-evals: proposed improvement from eval transcript") diff --git a/evals/agent_evals/runner.py b/evals/agent_evals/runner.py new file mode 100644 index 000000000..b9d1f4d3f --- /dev/null +++ b/evals/agent_evals/runner.py @@ -0,0 +1,442 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Run agent evals: before-and-after comparisons over any material. + +An eval file is a JSON array of cases. Each case gives an agent a goal and grades +the outcome against a rubric: + + [ + { + "id": "list_farms", + "prompt": "List the Deadline Cloud farms and report how many there are.", + "tools": ["Bash"], + "rubric": "The answer states the number of farms.", + "materials": {"guide.md": "...optional reference text..."}, + "max_turns": 20, + "k": 3 + } + ] + +Each run launches an ISOLATED headless agent in a fresh sandbox; an LLM judge +grades the final answer against the rubric. With --revised-ref, every case runs +twice -- baseline at the current subject ref, candidate at the revised ref -- and +the summary reports the paired delta plus the subject diff (the PR-ready patch). + +Usage: + python -m agent_evals.runner run examples/cli_basics.json + python -m agent_evals.runner run my_eval.json --k 3 --revised-ref my-branch +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import statistics +import subprocess +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +from . import judge, subject as subject_mod +from .harness import HarnessError, run_agent + +OUTPUT_ROOT = Path(__file__).resolve().parents[1] / "output" + +DEFAULT_TOOLS = ["Bash", "Read", "Write", "Edit"] + +# A case with "env": "real_aws" submits real, billable jobs to a real farm. Such +# cases are SKIPPED unless the operator passes --allow-real-aws, and the farm/queue +# come from these env vars -- never hardcoded, so a public eval file names no +# account. Prompts may reference {farm_id}/{queue_id}/{region}, filled from here. +REAL_AWS_ENV = "real_aws" +ENV_FARM_ID = "DEADLINE_EVAL_FARM_ID" +ENV_QUEUE_ID = "DEADLINE_EVAL_QUEUE_ID" +ENV_REGION = "DEADLINE_EVAL_REGION" + + +class RealAwsConfigError(RuntimeError): + """Raised when a real_aws case is requested but the environment isn't ready.""" + + +def _real_aws_context() -> dict: + """Farm/queue/region for real_aws cases, from env vars. Raises if the required + ones are unset -- we never fall back to a hardcoded or ambient default farm.""" + farm = os.environ.get(ENV_FARM_ID, "").strip() + queue = os.environ.get(ENV_QUEUE_ID, "").strip() + missing = [n for n, v in ((ENV_FARM_ID, farm), (ENV_QUEUE_ID, queue)) if not v] + if missing: + raise RealAwsConfigError( + f"real_aws case needs {' and '.join(missing)} set to a NON-PRODUCTION " + "sandbox farm/queue you own (these submit real, billable jobs)." + ) + return {"farm_id": farm, "queue_id": queue, "region": os.environ.get(ENV_REGION, "").strip()} + + +def _aws_authenticated() -> bool: + """True when `deadline auth status` reports the API reachable.""" + try: + out = subprocess.run( + ["deadline", "auth", "status"], capture_output=True, text=True, timeout=30 + ).stdout + except (OSError, subprocess.TimeoutExpired): + return False + return '"api_availability": true' in out or "API Availability: True" in out + + +def _real_aws_skip_reason(allow_real_aws: bool) -> Optional[str]: + """Why a real_aws case should be skipped, or None if it's clear to run. + + Skipping (rather than failing) keeps a default `run` green: real_aws cases are + opt-in, submit billable jobs, and need live auth + a configured sandbox farm. + """ + if not allow_real_aws: + return "requires --allow-real-aws (submits real, billable jobs)" + try: + _real_aws_context() + except RealAwsConfigError as e: + return str(e) + if not _aws_authenticated(): + return "deadline auth status is not authenticated; run `deadline auth login`" + return None + + +# Telemetry shape for a run that never produced a result (harness error). Mirrors +# RunResult.telemetry_dict so aggregation treats it like any other failed run. +_EMPTY_TELEMETRY = { + "success": False, + "subtype": "harness_error", + "tool_calls": [], + "tool_call_count": 0, + "num_turns": 0, + "total_cost_usd": 0.0, + "duration_ms": 0, + "final_text": "", +} + + +def _write_failed_run(run_dir: Path, detail: str) -> None: + """Persist a harness-error run so its artifact exists alongside the others.""" + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "result.json").write_text( + json.dumps({**_EMPTY_TELEMETRY, "passed": False, "detail": detail}, indent=2) + ) + + +def _safe_write(base: Path, rel: str, content: str) -> None: + """Write content to base/rel, creating parent dirs. Rejects absolute paths and + '..' segments so a material/subject key can never write outside the sandbox.""" + rel_path = Path(rel) + if rel_path.is_absolute() or ".." in rel_path.parts: + raise ValueError(f"unsafe sandbox path: {rel!r}") + dest = base / rel_path + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(content) + + +def _subject_files(subj) -> dict: + """The subject's owned files (per its pathspec) at the CURRENTLY checked-out ref, + as {relative_path: content}. Used to seed read-material subjects into the + sandbox so a ref swap actually changes what the agent sees.""" + listing = subj._git_checked("ls-files", "-z", "--", subj.diff_pathspec).stdout + files = {} + for rel in filter(None, listing.split("\0")): + p = subj.root / rel + if p.is_file(): + files[rel] = p.read_text(errors="replace") + return files + + +def _run_case( + case: dict, run_dir: Path, model: Optional[str], subject_files=None, aws_ctx=None +) -> dict: + """One agent run + judge verdict; artifacts under run_dir. + + `aws_ctx` (farm_id/queue_id/region) is filled into {placeholders} in the prompt + and rubric for real_aws cases; None for offline/mock cases. + """ + fmt = dict(aws_ctx or {}) + prompt = case["prompt"].format(**fmt) if fmt else case["prompt"] + rubric = case["rubric"].format(**fmt) if fmt else case["rubric"] + materials = case.get("materials", {}) + if materials: + listing = ", ".join(f"materials/{name}" for name in materials) + prompt = f"{prompt}\n\nReference material is available in this directory: {listing}" + if subject_files: + prompt = f"{prompt}\n\nThe material under evaluation is available under subject/ in this directory." + + workdir = Path(tempfile.mkdtemp(prefix=f"eval-{case['id']}-")) + try: + for name, content in materials.items(): + _safe_write(workdir / "materials", name, content) + if subject_files: + # Seed the subject's owned files (docs, etc.) into the sandbox. Without + # this, a read-material A/B would compare two identical sandboxes: the + # ref swap happens in the repo checkout the sandboxed agent can't see. + for rel, content in subject_files.items(): + _safe_write(workdir / "subject", rel, content) + + try: + result = run_agent( + prompt, + workdir, + case.get("tools", DEFAULT_TOOLS), + max_turns=case.get("max_turns", 20), + model=model, + ) + except HarnessError as e: + # A run that could not launch or timed out fails just this run -- it must + # not abort the batch, and must not be scored as a pass. + _write_failed_run(run_dir, f"harness error: {e}") + print(f" run: FAIL (harness error) -- {e}") + return {**_EMPTY_TELEMETRY, "passed": False, "detail": f"harness error: {e}"} + finally: + shutil.rmtree(workdir, ignore_errors=True) + + try: + verdict = judge.judge_answer(rubric, prompt, result.final_text, model=model) + passed, reasoning = verdict.passed, verdict.reasoning + except judge.JudgeError as e: + # A judge that can't render a verdict must not silently pass a run. + passed, reasoning = False, f"judge error: {e}" + + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "result.json").write_text( + json.dumps({**result.telemetry_dict(), "passed": passed, "detail": reasoning}, indent=2) + ) + with (run_dir / "events.jsonl").open("w") as f: + for ev in result.raw_events: + f.write(json.dumps(ev) + "\n") + + mark = "PASS" if passed else "FAIL" + print( + f" run: {mark} (tools={result.tool_call_count}, turns={result.num_turns}, " + f"cost=${result.total_cost_usd:.4f}) -- {reasoning}" + ) + return {**result.telemetry_dict(), "passed": passed, "detail": reasoning} + + +def _deadline_provenance() -> str: + """A one-line description of the `deadline` the agent will drive: path, version, + and whether it's an editable install (points at a source checkout). Surfaced at + startup so an operator never unknowingly evaluates a stale or fork-shadowed CLI + -- the 'editable install shadows your real deadline' footgun.""" + path = shutil.which("deadline") + if not path: + return "deadline: NOT ON PATH (agent runs that need it will fail)" + try: + version = subprocess.run( + [path, "--version"], capture_output=True, text=True, timeout=30 + ).stdout.strip() + except (OSError, subprocess.TimeoutExpired): + version = "version unknown" + editable = "" + try: + # A console-script shebang names the interpreter; ask pip where the package + # lives and whether it's an editable (source-checkout) install. + interp = Path(path).read_text().splitlines()[0].lstrip("#!").strip() + show = subprocess.run( + [interp, "-m", "pip", "show", "deadline"], capture_output=True, text=True, timeout=30 + ).stdout + for line in show.splitlines(): + if line.lower().startswith("editable project location"): + editable = f" [EDITABLE -> {line.split(':', 1)[1].strip()}]" + except (OSError, subprocess.TimeoutExpired, IndexError, ValueError): + # Provenance is a diagnostic, never a gate: if the launcher isn't a readable + # text stub (a native `deadline.exe` on Windows raises UnicodeDecodeError, a + # ValueError subclass) or pip can't be reached, report without the editable + # detail rather than aborting the run before any eval executes. + pass + return f"deadline: {version} at {path}{editable}" + + +def _aggregate(runs: list) -> dict: + if not runs: + return {"pass_rate": 0.0, "median_turns": 0.0, "median_cost_usd": 0.0, "n": 0} + return { + "pass_rate": sum(1 for r in runs if r["passed"]) / len(runs), + "median_turns": statistics.median(float(r["num_turns"]) for r in runs), + "median_cost_usd": statistics.median(r["total_cost_usd"] for r in runs), + "n": len(runs), + } + + +def _cmd_run(args: argparse.Namespace) -> int: + cases = json.loads(Path(args.eval_file).read_text()) + if not isinstance(cases, list): + print("ERROR: eval file must be a JSON array of cases.") + return 2 + + if args.seed_subject and not args.revised_ref: + print("ERROR: --seed-subject only applies in A/B mode; pass --revised-ref too.") + return 2 + + # Always report which deadline CLI the agent will drive -- guards against + # silently evaluating a stale or fork-shadowed editable install. + print(f"[env] {_deadline_provenance()}") + + subj = None + base_ref = None + if args.revised_ref: + subj = subject_mod.repo_subject(args.pathspec) + base = subj._git("rev-parse", "--abbrev-ref", "HEAD").stdout.strip() + base_ref = args.base_ref or base + # Fail fast, BEFORE any (costly) agent runs: both refs must resolve, and the + # operator's working tree must be clean -- checkout() discards changes under + # the pathspec, and that must never eat uncommitted work. + try: + subj.assert_clean() + for ref in (base_ref, args.revised_ref): + subj._git_checked("rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}") + except subject_mod.SubjectError as e: + print(f"ERROR: {e}") + return 2 + print(f"[subject] {subj.root} (pathspec={subj.diff_pathspec})") + print(f"[variants] baseline={base_ref} revised={args.revised_ref}") + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + out_root = OUTPUT_ROOT / Path(args.eval_file).stem / timestamp + + summaries = [] + exit_code = 0 + try: + for case in cases: + k = case.get("k", args.k) + + # real_aws cases submit real, billable jobs: skip (don't fail) unless the + # operator opted in, the farm/queue env vars are set, and auth is live. + # Skipping keeps a plain `run` green in CI while these are opt-in only. + aws_ctx = None + if case.get("env") == REAL_AWS_ENV: + skip = _real_aws_skip_reason(args.allow_real_aws) + if skip: + print(f"\n=== {case['id']} === SKIPPED (real_aws): {skip}") + summaries.append({"case_id": case["id"], "skipped": skip}) + continue + aws_ctx = _real_aws_context() + + print(f"\n=== {case['id']} (k={k}) ===") + variants = {} + + refs = ( + {"baseline": base_ref, "revised": args.revised_ref} if subj else {"baseline": None} + ) + for variant, ref in refs.items(): + subject_files = None + if subj and ref: + subj.checkout(ref) + print(f" [{variant}] subject at {ref}") + if args.seed_subject: + # Read-material subject (docs): the sandboxed agent can't see + # the repo checkout, so seed the owned files AT THIS REF into + # the sandbox -- otherwise both variants get identical input + # and the A/B silently reports no_change. + subject_files = _subject_files(subj) + runs = [ + _run_case( + case, + out_root / case["id"] / variant / f"run-{i}", + args.model, + subject_files, + aws_ctx, + ) + for i in range(1, k + 1) + ] + variants[variant] = {"aggregate": _aggregate(runs), "runs": runs} + + summary = {"case_id": case["id"], "variants": variants} + if subj and args.revised_ref: + summary["source_diff"] = subj.diff_refs(base_ref, args.revised_ref) + b, c = variants["baseline"]["aggregate"], variants["revised"]["aggregate"] + improved = c["pass_rate"] > b["pass_rate"] or ( + c["pass_rate"] == b["pass_rate"] and c["median_turns"] < b["median_turns"] + ) + regressed = c["pass_rate"] < b["pass_rate"] + summary["verdict"] = ( + "improved" if improved else "regressed" if regressed else "no_change" + ) + print( + f" [A/B] pass {b['pass_rate']:.0%} -> {c['pass_rate']:.0%}, " + f"turns {b['median_turns']:g} -> {c['median_turns']:g}: {summary['verdict']}" + ) + if regressed: + exit_code = 4 + summaries.append(summary) + finally: + if subj and base_ref: + subj.checkout(base_ref) # always leave the checkout on baseline + + out_root.mkdir(parents=True, exist_ok=True) + (out_root / "summary.json").write_text(json.dumps(summaries, indent=2)) + print(f"\nsummary -> {out_root / 'summary.json'}") + + if subj and args.revised_ref: + # The diff is identical across A/B cases, so take it from any case that has + # one -- summaries[0] may be a skipped real_aws case with no source_diff, + # which would silently suppress a genuinely earned proposal. + diff = next((s["source_diff"] for s in summaries if s.get("source_diff")), "") + if _should_emit_proposal(summaries, diff): + (out_root / "proposal.patch").write_text(diff) + print(f"proposal -> {out_root / 'proposal.patch'}") + elif any(s.get("verdict") == "regressed" for s in summaries): + print("[proposal] skipped: the change regressed at least one eval.") + return exit_code + + +def _should_emit_proposal(summaries: list, diff: str) -> bool: + """Emit the PR-ready patch only when the change measurably improved at least + one eval AND regressed none. The patch is the whole base..revised diff, so a + change that breaks any eval must never be surfaced as ready to ship.""" + if not diff.strip(): + return False + verdicts = [s.get("verdict") for s in summaries] + return "improved" in verdicts and "regressed" not in verdicts + + +def main(argv: Optional[list] = None) -> int: + ap = argparse.ArgumentParser(prog="agent-evals") + sub = ap.add_subparsers(dest="cmd") + + run = sub.add_parser("run", help="run an eval file") + run.add_argument("eval_file", help="path to a JSON eval file") + run.add_argument("--k", type=int, default=1, help="runs per case per variant") + run.add_argument("--model", help="model for the agent AND the judge") + run.add_argument( + "--revised-ref", + help="A/B mode: also run with this repo at the given git ref and compare", + ) + run.add_argument("--base-ref", help="baseline ref for A/B (default: current branch)") + run.add_argument( + "--pathspec", + default="src", + help="repo paths the A/B owns, e.g. 'src' or ':(glob)docs/**/*.md'", + ) + run.add_argument( + "--seed-subject", + action="store_true", + help="copy the subject's owned files into each run's sandbox (under subject/) " + "so the agent READS them -- required for docs/prose A/B, where the agent has " + "no path to the repo checkout. Not needed for the CLI-source case (pip install " + "-e makes the ref swap take effect through the installed `deadline`).", + ) + run.add_argument( + "--allow-real-aws", + action="store_true", + help=f'opt in to running cases with "env": "{REAL_AWS_ENV}", which submit ' + f"REAL, BILLABLE jobs. Requires {ENV_FARM_ID}/{ENV_QUEUE_ID} (optionally " + f"{ENV_REGION}) set to a non-production sandbox you own, and `deadline auth " + "login`. Without this flag such cases are skipped.", + ) + + args = ap.parse_args(argv) + if args.cmd == "run": + return _cmd_run(args) + ap.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/evals/agent_evals/subject.py b/evals/agent_evals/subject.py new file mode 100644 index 000000000..f009e832a --- /dev/null +++ b/evals/agent_evals/subject.py @@ -0,0 +1,136 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""The subject under test: the material an eval A/B-compares and improves. + +A Subject is any git checkout whose revisions can be A/B-tested: baseline = the +subject at one ref, candidate = the subject at another (usually an agent's edits +committed to a scratch ref). The diff between them, scoped to the paths the +subject owns, is the PR-ready proposed improvement. + +Two ready-made subjects cover the common cases: + - repo_subject(): this deadline-cloud checkout itself -- A/B changes to the CLI + source or the repo docs. Requires an editable install (`pip install -e .`) so + the `deadline` the agent drives IS this checkout. + - corpus_subject(markdown): any fetched material (an AWS docs page, a blog post, + a web-search result) seeded into a throwaway git repo so it can be diffed and + revised like everything else. +""" + +from __future__ import annotations + +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path + +BASE_REF = "eval-base" + +# The doc filename used when seeding fetched material into a corpus. +CORPUS_DOC = "source.md" + + +class SubjectError(RuntimeError): + """Raised when a git operation on the subject fails or would lose work.""" + + +@dataclass +class Subject: + """A git checkout + the pathspec its evals own (scopes diffs and cleanup).""" + + root: Path + diff_pathspec: str = "." + + def _git(self, *args: str) -> subprocess.CompletedProcess: + return subprocess.run(["git", "-C", str(self.root), *args], capture_output=True, text=True) + + def _git_checked(self, *args: str) -> subprocess.CompletedProcess: + """Like _git, but raises on failure. Used where a silent git failure would + corrupt the experiment -- e.g. a checkout that never happened leaves the + 'revised' variant running baseline source and reporting a bogus no-delta.""" + proc = self._git(*args) + if proc.returncode != 0: + raise SubjectError( + f"git {' '.join(args)} failed in {self.root}: {proc.stderr.strip()[:300]}" + ) + return proc + + def assert_clean(self) -> None: + """Refuse to operate on a checkout with uncommitted work under the owned paths. + + checkout()/reset_clean() discard working-tree changes under the pathspec by + design (one variant must not bleed into the next) -- but the operator's own + uncommitted work must never be what gets discarded. Callers invoke this once + before the first destructive operation; a dirty tree is the operator's to + stash or commit, not ours to delete. + """ + status = self._git_checked("status", "--porcelain", "--", self.diff_pathspec).stdout.strip() + if status: + raise SubjectError( + f"the checkout at {self.root} has uncommitted changes under " + f"'{self.diff_pathspec}':\n{status[:500]}\n" + "Commit or stash them first -- running evals would discard them." + ) + + def checkout(self, ref: str) -> None: + """Put the checkout on `ref`, discarding working-tree edits first.""" + self.reset_clean() + self._git_checked("checkout", "-q", ref) + + def reset_clean(self) -> None: + """Discard uncommitted edits under the owned paths so one variant never + bleeds into the next. Scoped to diff_pathspec -- NOT a whole-tree + `git reset --hard` -- so an operator's uncommitted work outside the owned + paths is never destroyed. `checkout HEAD` reverts tracked (staged and + worktree) edits; `clean -fd` removes untracked files; both scoped.""" + self._git("checkout", "HEAD", "--", self.diff_pathspec) + self._git("clean", "-fd", self.diff_pathspec) + + def capture_diff(self) -> str: + """Uncommitted edits to the owned paths (including new files) as a unified + diff -- the candidate's proposed change.""" + self._git("add", "-N", self.diff_pathspec) + return self._git("diff", "--", self.diff_pathspec).stdout + + def diff_refs(self, base: str, revised: str) -> str: + """The committed change a candidate ref is testing, as a unified diff.""" + return self._git_checked("diff", f"{base}..{revised}", "--", self.diff_pathspec).stdout + + def commit_scratch(self, ref: str, message: str) -> str: + """Commit ONLY the owned paths to a fresh branch `ref` (recreated if it + exists) and return the ref. Scoped so stray edits elsewhere never land in a + proposal.""" + self._git_checked("checkout", "-B", ref) + self._git_checked("add", self.diff_pathspec) + self._git_checked("commit", "-m", message) + return ref + + +def repo_subject(pathspec: str = "src") -> Subject: + """This deadline-cloud checkout as the subject (default: the CLI source). + + Pass pathspec=":(glob)docs/**/*.md" to A/B the repo docs instead. + """ + root = Path(__file__).resolve().parents[2] + return Subject(root=root, diff_pathspec=pathspec) + + +def corpus_subject(markdown: str, dest: "Path | None" = None) -> Subject: + """Seed fetched material (docs page, blog post, search result) into a git repo + so it can be A/B-tested and revised. Returns the ready Subject on BASE_REF.""" + root = Path(dest) if dest else Path(tempfile.mkdtemp(prefix="eval-corpus-")) + root.mkdir(parents=True, exist_ok=True) + if any(root.iterdir()) and not (root / ".git").exists(): + raise RuntimeError(f"{root} is non-empty and not a git repo; refusing to seed over it.") + + subj = Subject(root=root, diff_pathspec=":(glob)**/*.md") + if not (root / ".git").exists(): + subj._git("init", "-q") + # Local identity so the commit works without global git config. + subj._git("config", "user.email", "agent-evals@amazon.com") + subj._git("config", "user.name", "agent-evals") + + subj._git("checkout", "-q", "-B", BASE_REF) + (root / CORPUS_DOC).write_text(markdown) + subj._git("add", CORPUS_DOC) + subj._git("commit", "-q", "-m", "seed corpus from fetched material") + return subj diff --git a/evals/examples/aws_cli.json b/evals/examples/aws_cli.json new file mode 100644 index 000000000..8f426900b --- /dev/null +++ b/evals/examples/aws_cli.json @@ -0,0 +1,9 @@ +[ + { + "id": "deadline_via_aws_cli", + "prompt": "Using only `aws deadline help` (do not call any AWS APIs), explain how you would list the farms in an account and then the queues in one farm with the AWS CLI. Name the exact subcommands and required parameters.", + "tools": ["Bash"], + "rubric": "The answer names `aws deadline list-farms` and `aws deadline list-queues` and states that list-queues requires --farm-id.", + "max_turns": 10 + } +] diff --git a/evals/examples/deadline_cli.json b/evals/examples/deadline_cli.json new file mode 100644 index 000000000..b98054549 --- /dev/null +++ b/evals/examples/deadline_cli.json @@ -0,0 +1,23 @@ +[ + { + "id": "discover_submit_workflow", + "prompt": "Using only `deadline --help` and its subcommand help pages (do not run any other deadline commands), explain the full workflow to submit a job bundle, wait for it to finish, and download its output. Name the exact commands and flags you would use.", + "tools": ["Bash"], + "rubric": "The answer names `deadline bundle submit` for submission and identifies real commands/flags for waiting on completion and downloading output (for example `deadline job download-output`). It must not invent flags that do not appear in the help text.", + "max_turns": 15 + }, + { + "id": "explain_farm_queue_defaults", + "prompt": "Using only `deadline config --help` and `deadline job get --help`, explain whether --farm-id and --queue-id are required on `deadline job get`, and where their values come from when omitted.", + "tools": ["Bash"], + "rubric": "The answer states that the id options are optional and fall back to configured defaults (deadline config / defaults.farm_id and defaults.queue_id).", + "max_turns": 10 + }, + { + "id": "what_is_openjd", + "prompt": "Using only `deadline bundle --help`, `deadline bundle submit --help`, and any references to Open Job Description you find in the CLI help text, explain what Open Job Description is and what kinds of workloads it supports.", + "tools": ["Bash"], + "rubric": "The answer describes Open Job Description as a general-purpose compute job template specification (or words to that effect) — not as something limited to rendering, visual effects, or visual compute specifically. It may mention rendering as one example use case, but must not frame OpenJD as exclusively or primarily for visual/render workloads.", + "max_turns": 10 + } +] diff --git a/evals/examples/docs_page.json b/evals/examples/docs_page.json new file mode 100644 index 000000000..5dd5b8ea4 --- /dev/null +++ b/evals/examples/docs_page.json @@ -0,0 +1,12 @@ +[ + { + "id": "follow_bundle_guide", + "prompt": "Using ONLY the reference material provided (do not run deadline or aws commands), write out the exact sequence of commands a user should run to create and submit the job bundle the guide describes. If the guide is missing a step or prerequisite a user would need, say so explicitly.", + "tools": ["Read"], + "rubric": "The answer reproduces a correct command sequence from the guide, and explicitly calls out any step the guide leaves ambiguous or missing rather than inventing details the guide does not contain.", + "materials": { + "guide.md": "# Submitting your first job bundle\n\nA job bundle pairs an Open Job Description template with the files your job needs.\n\n1. Create a directory named `my_bundle` containing a `template.yaml`.\n2. Submit it with `deadline bundle submit my_bundle`.\n3. Watch progress in the Deadline Cloud monitor.\n" + }, + "max_turns": 8 + } +] diff --git a/evals/examples/real_aws_submit.json b/evals/examples/real_aws_submit.json new file mode 100644 index 000000000..530e69796 --- /dev/null +++ b/evals/examples/real_aws_submit.json @@ -0,0 +1,14 @@ +[ + { + "id": "submit_job_end_to_end", + "env": "real_aws", + "prompt": "There is a Deadline Cloud job bundle in the ./materials/bundle/ directory (an Open Job Description template that runs a short echo task). Using the `deadline` CLI, submit this bundle to farm {farm_id} and queue {queue_id}, wait for the job to finish, and report its final lifecycle/task-run status. Use `deadline bundle submit ./materials/bundle --farm-id {farm_id} --queue-id {queue_id} --yes` and then wait for completion (for example with `deadline job wait` or by polling `deadline job get`). State the final status clearly in your answer.", + "tools": ["Bash", "Read"], + "rubric": "A passing answer shows the agent submitted the bundle with `deadline bundle submit` and obtained a job id, then waited for and reported the job's terminal status. The job must have reached SUCCEEDED (task run status SUCCEEDED / lifecycle status UPDATE_SUCCEEDED). An answer that only submitted without confirming completion, or where the job did not succeed, fails.", + "max_turns": 40, + "k": 1, + "materials": { + "bundle/template.yaml": "specificationVersion: 'jobtemplate-2023-09'\nname: agent-eval-echo\ndescription: Minimal echo job for agent-evals real_aws smoke test\nsteps:\n - name: Echo\n script:\n actions:\n onRun:\n command: '{{Task.File.run}}'\n embeddedFiles:\n - name: run\n filename: run.sh\n type: TEXT\n runnable: true\n data: |\n #!/bin/bash\n echo \"agent-evals real_aws smoke test: hello from Deadline Cloud\"\n" + } + } +] diff --git a/evals/tests/test_harness.py b/evals/tests/test_harness.py new file mode 100644 index 000000000..369686b2d --- /dev/null +++ b/evals/tests/test_harness.py @@ -0,0 +1,26 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Harness launch-error handling (no real agent calls).""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from agent_evals.harness import HarnessError, run_agent # noqa: E402 + + +def test_run_agent_raises_harness_error_on_missing_binary(tmp_path: Path) -> None: + # A missing agent binary must surface as a descriptive HarnessError, not a raw + # FileNotFoundError that aborts the whole batch. + with pytest.raises(HarnessError, match="could not launch"): + run_agent( + "hello", + tmp_path, + ["Bash"], + claude_bin="definitely-not-a-real-binary-xyz", + ) diff --git a/evals/tests/test_judge_and_runner.py b/evals/tests/test_judge_and_runner.py new file mode 100644 index 000000000..0c3be7702 --- /dev/null +++ b/evals/tests/test_judge_and_runner.py @@ -0,0 +1,241 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Judge verdict parsing + runner aggregation (no live model calls).""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from agent_evals import judge, reviser, runner # noqa: E402 +from agent_evals.subject import BASE_REF, CORPUS_DOC, corpus_subject # noqa: E402 + + +def test_extract_verdict_plain_json() -> None: + v = judge._extract_verdict('{"passed": true, "reasoning": "meets the rubric"}') + assert v.passed is True and "rubric" in v.reasoning + + +def test_extract_verdict_tolerates_code_fence() -> None: + v = judge._extract_verdict('```json\n{"passed": false, "reasoning": "missing"}\n```') + assert v.passed is False + + +def test_extract_verdict_rejects_no_json() -> None: + with pytest.raises(judge.JudgeError): + judge._extract_verdict("I think it passes.") + + +def test_extract_verdict_requires_passed_key() -> None: + with pytest.raises(judge.JudgeError): + judge._extract_verdict('{"reasoning": "no verdict"}') + + +def test_extract_verdict_string_false_is_fail() -> None: + # bool("false") is True in Python — the coercion must handle string forms safely. + v = judge._extract_verdict('{"passed": "false", "reasoning": "did not meet rubric"}') + assert v.passed is False + + +def test_extract_verdict_string_true_is_pass() -> None: + v = judge._extract_verdict('{"passed": "true", "reasoning": "ok"}') + assert v.passed is True + + +def test_extract_verdict_ambiguous_value_raises() -> None: + with pytest.raises(judge.JudgeError, match="ambiguous"): + judge._extract_verdict('{"passed": "maybe", "reasoning": "unsure"}') + + +def test_empty_answer_fails_without_model_call() -> None: + v = judge.judge_answer("rubric", "prompt", " ") + assert v.passed is False + + +def test_aggregate_medians_and_pass_rate() -> None: + runs = [ + {"passed": True, "num_turns": 4, "total_cost_usd": 0.10}, + {"passed": False, "num_turns": 10, "total_cost_usd": 0.30}, + {"passed": True, "num_turns": 6, "total_cost_usd": 0.20}, + ] + agg = runner._aggregate(runs) + assert agg["pass_rate"] == pytest.approx(2 / 3) + assert agg["median_turns"] == 6.0 + assert agg["median_cost_usd"] == pytest.approx(0.20) + assert agg["n"] == 3 + + +def test_aggregate_empty() -> None: + assert runner._aggregate([]) == { + "pass_rate": 0.0, + "median_turns": 0.0, + "median_cost_usd": 0.0, + "n": 0, + } + + +def test_proposal_emitted_when_improved_and_none_regressed() -> None: + summaries = [{"verdict": "improved"}, {"verdict": "no_change"}] + assert runner._should_emit_proposal(summaries, "diff --git a b\n") is True + + +def test_proposal_suppressed_on_mixed_verdicts() -> None: + # The patch is the whole base..revised diff -- a change that regressed ANY + # eval must never be surfaced as PR-ready, even if it improved another. + summaries = [{"verdict": "improved"}, {"verdict": "regressed"}] + assert runner._should_emit_proposal(summaries, "diff --git a b\n") is False + + +def test_proposal_suppressed_without_improvement() -> None: + summaries = [{"verdict": "no_change"}, {"verdict": "no_change"}] + assert runner._should_emit_proposal(summaries, "diff --git a b\n") is False + + +def test_proposal_suppressed_on_empty_diff() -> None: + assert runner._should_emit_proposal([{"verdict": "improved"}], " \n") is False + + +def test_source_diff_found_past_skipped_case() -> None: + # A skipped real_aws case carries no source_diff. Picking summaries[0] blindly + # would yield "" and silently suppress a proposal a later case earned. + summaries = [ + {"case_id": "real_aws", "skipped": "requires --allow-real-aws"}, + {"case_id": "docs", "verdict": "improved", "source_diff": "diff --git a b\n"}, + ] + diff = next((s["source_diff"] for s in summaries if s.get("source_diff")), "") + assert diff == "diff --git a b\n" + assert runner._should_emit_proposal(summaries, diff) is True + + +def test_subject_files_reflects_checked_out_ref(tmp_path) -> None: + # The seeded files must track the CURRENT ref, so baseline and revised runs get + # different content -- otherwise a docs A/B compares identical sandboxes. + subj = corpus_subject("# Guide\nbaseline text\n", tmp_path / "corpus") + base_files = runner._subject_files(subj) + assert base_files[CORPUS_DOC] == "# Guide\nbaseline text\n" + + (subj.root / CORPUS_DOC).write_text("# Guide\nrevised text\n") + revised_ref = subj.commit_scratch("revised", "revise") + subj.checkout(revised_ref) + assert runner._subject_files(subj)[CORPUS_DOC] == "# Guide\nrevised text\n" + + +def test_subject_files_scoped_to_pathspec(tmp_path) -> None: + # corpus_subject owns *.md; a non-markdown file must not be seeded. + subj = corpus_subject("# Guide\n", tmp_path / "corpus") + (subj.root / "notes.txt").write_text("not markdown") + subj._git("add", "-A") + subj._git("commit", "-q", "-m", "add non-md") + files = runner._subject_files(subj) + assert CORPUS_DOC in files + assert "notes.txt" not in files + + +def test_safe_write_creates_nested_dirs(tmp_path) -> None: + # A material/subject key with a path separator must create parent dirs, not + # raise FileNotFoundError. + runner._safe_write(tmp_path / "materials", "docs/guide.md", "content") + assert (tmp_path / "materials" / "docs" / "guide.md").read_text() == "content" + + +def test_safe_write_rejects_parent_traversal(tmp_path) -> None: + with pytest.raises(ValueError, match="unsafe"): + runner._safe_write(tmp_path / "materials", "../escape.md", "x") + + +def test_safe_write_rejects_absolute_path(tmp_path) -> None: + with pytest.raises(ValueError, match="unsafe"): + runner._safe_write(tmp_path / "materials", "/etc/evil", "x") + + +def test_real_aws_context_requires_env(monkeypatch) -> None: + monkeypatch.delenv(runner.ENV_FARM_ID, raising=False) + monkeypatch.delenv(runner.ENV_QUEUE_ID, raising=False) + with pytest.raises(runner.RealAwsConfigError, match="FARM_ID"): + runner._real_aws_context() + + +def test_real_aws_context_reads_env(monkeypatch) -> None: + monkeypatch.setenv(runner.ENV_FARM_ID, "farm-x") + monkeypatch.setenv(runner.ENV_QUEUE_ID, "queue-y") + monkeypatch.setenv(runner.ENV_REGION, "eu-central-1") + ctx = runner._real_aws_context() + assert ctx == {"farm_id": "farm-x", "queue_id": "queue-y", "region": "eu-central-1"} + + +def test_real_aws_skipped_without_optin(monkeypatch) -> None: + monkeypatch.setenv(runner.ENV_FARM_ID, "farm-x") + monkeypatch.setenv(runner.ENV_QUEUE_ID, "queue-y") + reason = runner._real_aws_skip_reason(allow_real_aws=False) + assert reason and "allow-real-aws" in reason + + +def test_real_aws_skip_reason_flags_missing_env(monkeypatch) -> None: + monkeypatch.delenv(runner.ENV_FARM_ID, raising=False) + monkeypatch.delenv(runner.ENV_QUEUE_ID, raising=False) + reason = runner._real_aws_skip_reason(allow_real_aws=True) + assert reason and "FARM_ID" in reason + + +def test_provenance_survives_binary_launcher(tmp_path, monkeypatch) -> None: + # A native launcher (deadline.exe on Windows) isn't decodable UTF-8. Reading it + # raises UnicodeDecodeError (a ValueError, NOT an OSError) -- that must not abort + # the whole run before any eval executes. + launcher = tmp_path / "deadline" + launcher.write_bytes(b"\x7fELF\x02\x01\x01\x00\xff\xfe\xfd") + monkeypatch.setattr(runner.shutil, "which", lambda _: str(launcher)) + monkeypatch.setattr( + runner.subprocess, + "run", + lambda *a, **k: subprocess.CompletedProcess(a[0], 0, stdout="1.2.3", stderr=""), + ) + line = runner._deadline_provenance() + assert "1.2.3" in line and "EDITABLE" not in line + + +def test_judge_timeout_raises_judge_error(monkeypatch) -> None: + # A hung judge must fail its own run, not hang the batch -- the harness timeout + # can't cover the judge call. + def _timeout(*a, **k): + raise subprocess.TimeoutExpired(cmd="claude", timeout=k.get("timeout", 0)) + + monkeypatch.setattr(judge.subprocess, "run", _timeout) + with pytest.raises(judge.JudgeError, match="timeout"): + judge.judge_answer("rubric", "prompt", "an answer", timeout_s=1) + + +def test_revise_timeout_raises_revise_error(tmp_path, monkeypatch) -> None: + # The reviser drives a full edit session; a hung one must fail loudly instead of + # blocking with no ceiling. + subj = corpus_subject("# Guide\n", tmp_path / "corpus") + run_dir = tmp_path / "out" / "case" / "run-1" + run_dir.mkdir(parents=True) + (run_dir / "events.jsonl").write_text("") + + real_run = subprocess.run + + def _timeout(cmd, *a, **k): + # Only the agent launch times out; git calls must still work. + if "claude" in cmd[0]: + raise subprocess.TimeoutExpired(cmd=cmd[0], timeout=k.get("timeout", 0)) + return real_run(cmd, *a, **k) + + monkeypatch.setattr(reviser.subprocess, "run", _timeout) + with pytest.raises(reviser.ReviseError, match="timeout"): + reviser.revise(subj, run_dir, goal="g", base_ref=BASE_REF, timeout_s=1) + + +def test_revise_missing_binary_raises_revise_error(tmp_path) -> None: + subj = corpus_subject("# Guide\n", tmp_path / "corpus") + run_dir = tmp_path / "out" / "case" / "run-1" + run_dir.mkdir(parents=True) + (run_dir / "events.jsonl").write_text("") + with pytest.raises(reviser.ReviseError, match="could not launch"): + reviser.revise( + subj, run_dir, goal="g", base_ref=BASE_REF, claude_bin="definitely-not-real-xyz" + ) diff --git a/evals/tests/test_subject.py b/evals/tests/test_subject.py new file mode 100644 index 000000000..27a044781 --- /dev/null +++ b/evals/tests/test_subject.py @@ -0,0 +1,177 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Subject git mechanics on real temp repos (no agent calls).""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from agent_evals.subject import ( # noqa: E402 + BASE_REF, + CORPUS_DOC, + Subject, + SubjectError, + corpus_subject, +) + + +def _git_out(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), *args], capture_output=True, text=True, check=True + ).stdout + + +def test_corpus_subject_seeds_committed_repo(tmp_path: Path) -> None: + subj = corpus_subject("# Guide\nStep 1.\n", tmp_path / "corpus") + assert (subj.root / CORPUS_DOC).read_text().startswith("# Guide") + assert _git_out(subj.root, "rev-parse", "--abbrev-ref", "HEAD").strip() == BASE_REF + + +def test_corpus_subject_refuses_non_git_dir(tmp_path: Path) -> None: + dest = tmp_path / "corpus" + dest.mkdir() + (dest / "keep.txt").write_text("do not clobber") + with pytest.raises(RuntimeError): + corpus_subject("# New\n", dest) + + +def test_capture_diff_scopes_to_pathspec(tmp_path: Path) -> None: + subj = corpus_subject("# Guide\noriginal\n", tmp_path / "corpus") + (subj.root / CORPUS_DOC).write_text("# Guide\nimproved\n") + (subj.root / "notes.txt").write_text("not markdown") + diff = subj.capture_diff() + assert "improved" in diff + assert "notes.txt" not in diff # pathspec limits proposals to *.md + + +def test_scratch_commit_and_diff_refs(tmp_path: Path) -> None: + subj = corpus_subject("# Guide\noriginal\n", tmp_path / "corpus") + (subj.root / CORPUS_DOC).write_text("# Guide\nrevised\n") + ref = subj.commit_scratch("eval-revise-test", "test edit") + subj.checkout(BASE_REF) + assert "revised" in subj.diff_refs(BASE_REF, ref) + assert "revised" not in (subj.root / CORPUS_DOC).read_text() # back on baseline + + +def test_reset_clean_restores(tmp_path: Path) -> None: + subj = corpus_subject("# Guide\noriginal\n", tmp_path / "corpus") + (subj.root / CORPUS_DOC).write_text("broken edit") + subj.reset_clean() + assert (subj.root / CORPUS_DOC).read_text() == "# Guide\noriginal\n" + assert subj.capture_diff() == "" + + +def test_subject_dataclass_on_plain_repo(tmp_path: Path) -> None: + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "-C", str(repo), "init", "-q"], check=True) + subprocess.run(["git", "-C", str(repo), "config", "user.email", "t@example.com"], check=True) + subprocess.run(["git", "-C", str(repo), "config", "user.name", "t"], check=True) + (repo / "a.md").write_text("hello\n") + subprocess.run(["git", "-C", str(repo), "add", "-A"], check=True) + subprocess.run(["git", "-C", str(repo), "commit", "-q", "-m", "init"], check=True) + + subj = Subject(root=repo, diff_pathspec=".") + (repo / "a.md").write_text("hello world\n") + assert "world" in subj.capture_diff() + + +def _two_dir_repo(tmp_path: Path) -> Path: + """A committed repo with src/ and docs/ so pathspec scoping can be exercised.""" + repo = tmp_path / "repo" + (repo / "src").mkdir(parents=True) + (repo / "docs").mkdir() + (repo / "src" / "a.py").write_text("orig code\n") + (repo / "docs" / "b.md").write_text("orig doc\n") + subprocess.run(["git", "-C", str(repo), "init", "-q"], check=True) + subprocess.run(["git", "-C", str(repo), "config", "user.email", "t@example.com"], check=True) + subprocess.run(["git", "-C", str(repo), "config", "user.name", "t"], check=True) + subprocess.run(["git", "-C", str(repo), "add", "-A"], check=True) + subprocess.run(["git", "-C", str(repo), "commit", "-q", "-m", "init"], check=True) + return repo + + +def test_reset_clean_preserves_work_outside_pathspec(tmp_path: Path) -> None: + # reset_clean is scoped: an operator A/Bing `src` while carrying uncommitted + # edits to docs/ must NOT have those edits destroyed (the bug an unscoped + # `git reset --hard HEAD` would cause). + repo = _two_dir_repo(tmp_path) + subj = Subject(root=repo, diff_pathspec="src") + (repo / "src" / "a.py").write_text("variant edit\n") + (repo / "src" / "extra.py").write_text("untracked variant file\n") + (repo / "docs" / "b.md").write_text("OPERATOR WIP\n") + + subj.reset_clean() + + assert (repo / "src" / "a.py").read_text() == "orig code\n" # tracked edit reverted + assert not (repo / "src" / "extra.py").exists() # untracked removed + assert (repo / "docs" / "b.md").read_text() == "OPERATOR WIP\n" # out-of-pathspec preserved + + +def test_reset_clean_reverts_staged_edits_in_pathspec(tmp_path: Path) -> None: + # A staged (git add'd) edit under the pathspec must also be reverted. + repo = _two_dir_repo(tmp_path) + subj = Subject(root=repo, diff_pathspec="src") + (repo / "src" / "a.py").write_text("staged edit\n") + subprocess.run(["git", "-C", str(repo), "add", "src/a.py"], check=True) + + subj.reset_clean() + + assert (repo / "src" / "a.py").read_text() == "orig code\n" + staged = subprocess.run( + ["git", "-C", str(repo), "diff", "--cached", "--name-only"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + assert staged == "" + + +def test_checkout_raises_on_missing_ref(tmp_path: Path) -> None: + # A checkout that silently fails would leave the "revised" variant running + # baseline source, producing a plausible-looking but meaningless A/B result. + subj = corpus_subject("# Guide\n", tmp_path / "corpus") + with pytest.raises(SubjectError, match="checkout"): + subj.checkout("no-such-ref") + + +def test_diff_refs_raises_on_missing_ref(tmp_path: Path) -> None: + subj = corpus_subject("# Guide\n", tmp_path / "corpus") + with pytest.raises(SubjectError): + subj.diff_refs(BASE_REF, "no-such-ref") + + +def test_assert_clean_refuses_dirty_tree(tmp_path: Path) -> None: + # Uncommitted operator work under the pathspec must never be discarded. + subj = corpus_subject("# Guide\noriginal\n", tmp_path / "corpus") + (subj.root / CORPUS_DOC).write_text("uncommitted operator edit\n") + with pytest.raises(SubjectError, match="uncommitted"): + subj.assert_clean() + + +def test_assert_clean_flags_untracked_files(tmp_path: Path) -> None: + # git clean -fd would delete untracked files with no prompt; assert_clean must + # catch them, not just tracked modifications. + subj = corpus_subject("# Guide\n", tmp_path / "corpus") + (subj.root / "wip_notes.md").write_text("not yet committed\n") + with pytest.raises(SubjectError, match="uncommitted"): + subj.assert_clean() + + +def test_assert_clean_passes_on_clean_tree(tmp_path: Path) -> None: + subj = corpus_subject("# Guide\n", tmp_path / "corpus") + subj.assert_clean() # must not raise + + +def test_assert_clean_ignores_dirt_outside_pathspec(tmp_path: Path) -> None: + # Dirt outside the owned paths is none of our business -- the destructive + # clean is scoped to diff_pathspec, so the guard is too. + subj = corpus_subject("# Guide\n", tmp_path / "corpus") # owns only *.md + (subj.root / "scratch.txt").write_text("non-markdown dirt\n") + subj.assert_clean() # must not raise