diff --git a/.gitignore b/.gitignore
index 6ba89e0..204d124 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,3 +41,6 @@ Thumbs.db
# SQLite data directory (CHECKPOINT_BACKEND=sqlite)
data/
+
+# Eval per-run output — per-machine artifacts, never check in
+demos/eval/results/
diff --git a/Makefile b/Makefile
index c0d79c6..0dec2cb 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: dev ui migrate test lint frontend-dev frontend-build help
+.PHONY: dev ui migrate test lint frontend-dev frontend-build eval help
BACKEND := apps/backend
FRONTEND := apps/frontend
@@ -32,6 +32,12 @@ frontend-dev:
frontend-build:
cd $(FRONTEND) && npm install && npm run build
+# ── Eval (integration; real cloud + real LLM spend) ────────────────────────────
+# Needs the agent dev server running (make dev) and AWS / Azure creds for the
+# scenarios you're running. See demos/eval/README.md.
+eval:
+ python demos/eval/run.py
+
# ── Full stack (Docker Compose) ────────────────────────────────────────────────
compose-up:
docker compose -f deployment/docker-compose/docker-compose.yml up --build
@@ -43,4 +49,5 @@ compose-down:
help:
@echo "Backend: make dev | ui | migrate | test | lint | lint-fix | install"
@echo "Frontend: make frontend-dev | frontend-build"
+ @echo "Eval: make eval (integration — hits real cloud + LLM)"
@echo "Docker: make compose-up | compose-down"
diff --git a/README.md b/README.md
index c2da6b9..a92f8b7 100644
--- a/README.md
+++ b/README.md
@@ -5,6 +5,17 @@ Anthropic, OpenRouter, Groq, Gemini, Mistral, **Ollama for air-gapped / regulate
reuse your existing **Claude Code** subscription (auto-detected). Investigates incidents, finds root
causes, and gives actionable mitigation plans — without the cloud-vendor DevOps-agent price tag.
+> ### 📊 Benchmarked, not just demoed
+> On a **reproducible 10-incident suite** (real AWS + Azure resources, scored against ground truth), running on a **commodity open model** (`gpt-oss-120b` — no frontier model required):
+>
+> | Root causes found | Median time | Cost / investigation | vs. AWS DevOps Agent | vs. manual triage |
+> |:---:|:---:|:---:|:---:|:---:|
+> | **9 / 10 (90%)** | **~52 s** | **~$0.03** | **~10× cheaper**¹
(~$0.03 vs ~$0.43) | **~1,000× cheaper**² |
+>
+> *~$0.03 of compute replaces ~$50 of engineer toil — and costs a fraction of a managed cloud DevOps agent — while returning the answer in under a minute instead of half an hour.* Reproduce it with `make eval` → **[full benchmark & methodology](apps/documentation/benchmark.md)**.
+>
+> ¹ vs. AWS DevOps Agent — its per-second rate applied to the same wall-clock time (~$0.43/investigation); verify against AWS's published pricing. ² Illustrative unit economics vs. ~20–40 min of on-call triage. Cost shown is the provider-dashboard actual; see [caveats](apps/documentation/benchmark.md#caveats-read-these-before-quoting-numbers).
+
**Cloud setup:** [AWS (IAM)](apps/documentation/iam_setup.md) · [Azure (service principal / login)](apps/documentation/azure_setup.md)
## Demo
diff --git a/apps/backend/src/api/routers/chat.py b/apps/backend/src/api/routers/chat.py
index ddfa11a..990d325 100644
--- a/apps/backend/src/api/routers/chat.py
+++ b/apps/backend/src/api/routers/chat.py
@@ -60,6 +60,35 @@ def _clean(text: str) -> str:
return _CHANNEL_RE.sub("", text)
+def _extract_text(content: Any) -> str:
+ """Pull the user-visible text out of a chunk's ``content``, regardless of shape.
+
+ Reasoning / hybrid models (gpt-oss, OpenAI o1-family, some Anthropic configs,
+ Gemini Thinking) return ``content`` as a list of typed blocks:
+
+ [{'type': 'thinking', 'thinking': '...internal reasoning...'},
+ {'type': 'text', 'text': 'the actual answer'}]
+
+ Only the ``text`` blocks go to the UI; ``thinking`` blocks are billed but
+ not shown (they're shown via the tool-call inspector / reasoning UI if we
+ ever surface them). Plain-string content (older models, Sonnet, GPT-4o)
+ flows through unchanged.
+ """
+ if not content:
+ return ""
+ if isinstance(content, str):
+ return content
+ if isinstance(content, list):
+ parts: list[str] = []
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ txt = block.get("text") or ""
+ if isinstance(txt, str):
+ parts.append(txt)
+ return "".join(parts)
+ return ""
+
+
async def _stream_chat(session_id: str, user_message: str):
@@ -118,6 +147,13 @@ def _flush_text_buf():
await maybe_summarize(agent, config, session_id)
+ # DEBUG_LLM_CHUNKS=true in .env dumps every raw chunk's content + tool calls +
+ # usage to the log. Useful when a model returns "(no response)" in the UI to
+ # see whether the content is being stripped, wrapped in an unexpected format
+ # (e.g. gpt-oss harmony channels), or is genuinely empty from the provider.
+ import os as _os
+ _debug_chunks = _os.environ.get("DEBUG_LLM_CHUNKS", "").lower() in ("1", "true", "yes")
+
try:
async with asyncio.timeout(settings.investigation_timeout):
async for chunk, _meta in agent.astream(
@@ -129,6 +165,20 @@ def _flush_text_buf():
logger.info("[{sid}] cancelled by user", sid=sid)
yield f"data: {json.dumps({'type': 'cancelled'})}\n\n"
return
+ if _debug_chunks:
+ _raw = getattr(chunk, "content", None)
+ _raw_type = type(_raw).__name__
+ _raw_preview = repr(_raw)[:400] if _raw is not None else "None"
+ _tcc = getattr(chunk, "tool_call_chunks", None)
+ _tc = getattr(chunk, "tool_calls", None)
+ _um = getattr(chunk, "usage_metadata", None)
+ logger.info(
+ "[{sid}] RAW chunk type={t} content={c} tcc={tcc} tc={tc} usage={u}",
+ sid=sid, t=_raw_type, c=_raw_preview,
+ tcc=repr(_tcc)[:200] if _tcc else "-",
+ tc=repr(_tc)[:200] if _tc else "-",
+ u=repr(_um)[:200] if _um else "-",
+ )
um = getattr(chunk, "usage_metadata", None)
if um:
usage_meta = um
@@ -160,9 +210,10 @@ def _flush_text_buf():
}
content = getattr(chunk, "content", "")
- if content and isinstance(content, str):
+ visible = _extract_text(content)
+ if visible:
if not getattr(chunk, "tool_call_id", None):
- text_buf += content
+ text_buf += visible
new_clean = _clean(text_buf)
delta = new_clean[len(clean_buf):]
if delta:
@@ -199,13 +250,22 @@ def _flush_text_buf():
)
if isinstance(result, dict) and "error" in result:
logger.warning(" [{sid}] ⚠ error: {err}", sid=sid, err=result["error"])
- else:
- keys = list(result.keys()) if isinstance(result, dict) else type(result).__name__
+ elif isinstance(result, dict):
+ keys = list(result.keys())
count = result.get("count", result.get("events", result.get("functions", "")))
logger.info(
" [{sid}] ✓ keys={keys} count={count}",
sid=sid, keys=keys, count=count if isinstance(count, int) else "—",
)
+ else:
+ # gpt-oss / some providers return tool results as JSON arrays
+ # (e.g. CloudWatch metric data) — preserve them but don't try
+ # to .get() into a list. Log shape only.
+ logger.info(
+ " [{sid}] ✓ type={t} len={n}",
+ sid=sid, t=type(result).__name__,
+ n=len(result) if hasattr(result, "__len__") else "—",
+ )
tool_calls_log.append({"tool": call_info["tool"], "args": call_info["args"], "result": result})
label = _pick_label(call_info["tool"])
@@ -243,6 +303,17 @@ def _flush_text_buf():
if usage_meta:
usage["input_tokens"] = _field(usage_meta, "input_tokens", 0) or 0
usage["output_tokens"] = _field(usage_meta, "output_tokens", 0) or 0
+ # Capture reasoning tokens (gpt-oss, o1-family, Gemini thinking) for visibility
+ # in the cost card and downstream analysis. We intentionally do NOT add them
+ # to output_tokens for the cost calc here: providers differ on whether
+ # output_tokens already includes reasoning (gpt-oss: YES — 53 total includes
+ # 38 reasoning; Gemini: NO — but reasoning is also missing from the metadata,
+ # which is the actual undercount bug). Per-provider normalization is tracked
+ # in issue #59.
+ details = _field(usage_meta, "output_token_details", None) or {}
+ reasoning = _field(details, "reasoning", 0) or 0
+ if reasoning and reasoning > 0:
+ usage["reasoning_tokens"] = reasoning
cost = calc_cost(
usage["model"],
usage.get("input_tokens", 0),
diff --git a/apps/backend/tests/test_eval/test_salvage.py b/apps/backend/tests/test_eval/test_salvage.py
new file mode 100644
index 0000000..d0649a6
--- /dev/null
+++ b/apps/backend/tests/test_eval/test_salvage.py
@@ -0,0 +1,63 @@
+"""Unit tests for the runner's final-message salvage (demos/eval/run.py).
+
+The runner is integration-only, but ``_salvage_submit_json`` is a pure parser
+and the single thing standing between "model gave the right answer" and a FAIL
+when a model (notably gpt-oss) emits the ``submit_investigation`` payload as
+JSON text instead of a tool call. Pin its behavior so a contributor editing the
+brace-matching logic sees a clear failure.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+_EVAL_DIR = Path(__file__).resolve().parents[4] / "demos" / "eval"
+sys.path.insert(0, str(_EVAL_DIR))
+
+from run import _salvage_submit_json # noqa: E402
+
+
+def test_plain_json_dump_is_recovered():
+ """The common gpt-oss case: the whole message is the submit payload."""
+ payload = {
+ "root_cause_category": "SYSTEM_CHANGE",
+ "root_cause_summary": "bad image tag",
+ "evidence": ["ECS service reports desired=1, running=0"],
+ }
+ got = _salvage_submit_json(json.dumps(payload))
+ assert got == payload
+
+
+def test_json_wrapped_in_prose_and_fences_is_recovered():
+ """Some models wrap the JSON in explanatory prose and/or ``` fences."""
+ payload = {"root_cause_category": "RESOURCE_LIMIT", "evidence": ["x"]}
+ msg = f"Here is my analysis.\n```json\n{json.dumps(payload)}\n```\nDone."
+ assert _salvage_submit_json(msg) == payload
+
+
+def test_braces_and_quotes_inside_string_values_dont_break_matching():
+ """A traceback in the summary contains both quotes and braces — the
+ string-aware brace matcher must not miscount depth on those."""
+ payload = {
+ "root_cause_summary": 'KeyError at user = event["user_id"] with {braces}',
+ "root_cause_category": "COMPONENT_FAILURE",
+ "evidence": ["a", "b"],
+ }
+ msg = "Some prose. " + json.dumps(payload) + " trailing words."
+ assert _salvage_submit_json(msg) == payload
+
+
+def test_prose_without_schema_returns_empty():
+ """A fallback message ('see tool calls') must NOT be mistaken for an answer."""
+ assert _salvage_submit_json("Investigation complete. See tool calls for details.") == {}
+
+
+def test_non_schema_json_object_is_ignored():
+ """A JSON object that lacks the investigation keys is not a submit payload."""
+ assert _salvage_submit_json('{"foo": 1, "bar": [2, 3]}') == {}
+
+
+def test_empty_input_returns_empty():
+ assert _salvage_submit_json("") == {}
diff --git a/apps/backend/tests/test_eval/test_scoring.py b/apps/backend/tests/test_eval/test_scoring.py
new file mode 100644
index 0000000..9a758aa
--- /dev/null
+++ b/apps/backend/tests/test_eval/test_scoring.py
@@ -0,0 +1,263 @@
+"""Unit tests for the eval scorer (demos/eval/scoring.py).
+
+The runner itself is integration-only (hits real cloud), but the pure scoring
+functions are testable. Pin the scoring contract so a contributor who edits
+``scoring.py`` sees clear failures rather than mysterious eval regressions.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+_EVAL_DIR = Path(__file__).resolve().parents[4] / "demos" / "eval"
+sys.path.insert(0, str(_EVAL_DIR))
+
+from scoring import ( # noqa: E402
+ check_evidence_keywords,
+ check_root_cause_category,
+ check_services_affected,
+ check_tools_called,
+ score,
+)
+
+
+# ── individual checks ────────────────────────────────────────────────────────
+
+
+def test_root_cause_match_is_exact_and_case_insensitive():
+ ok, _ = check_root_cause_category("RESOURCE_LIMIT", "RESOURCE_LIMIT")
+ assert ok
+ ok, _ = check_root_cause_category("resource_limit", "RESOURCE_LIMIT")
+ assert ok
+
+
+def test_root_cause_mismatch_reports_both_values():
+ ok, msg = check_root_cause_category("RESOURCE_LIMIT", "COMPONENT_FAILURE")
+ assert not ok
+ assert "RESOURCE_LIMIT" in msg and "COMPONENT_FAILURE" in msg
+
+
+def test_root_cause_accepts_list_of_valid_categories():
+ """Many incidents are defensibly classifiable two ways (a crashing Lambda is
+ both COMPONENT_FAILURE and SYSTEM_CHANGE). Ground truth supports a list to
+ accept any of them — the scorer shouldn't punish a defensible call."""
+ ok, _ = check_root_cause_category("SYSTEM_CHANGE", ["COMPONENT_FAILURE", "SYSTEM_CHANGE"])
+ assert ok
+ ok, _ = check_root_cause_category("COMPONENT_FAILURE", ["COMPONENT_FAILURE", "SYSTEM_CHANGE"])
+ assert ok
+
+
+def test_root_cause_list_mismatch_lists_all_acceptable():
+ """When the agent picks an unacceptable category, the error message has to
+ show every accepted value — otherwise a contributor seeing the failure has
+ no idea what was allowed."""
+ ok, msg = check_root_cause_category(
+ "INPUT_ANOMALY", ["COMPONENT_FAILURE", "SYSTEM_CHANGE"]
+ )
+ assert not ok
+ assert "COMPONENT_FAILURE" in msg
+ assert "SYSTEM_CHANGE" in msg
+
+
+def test_services_match_strips_aws_amazon_prefixes():
+ """The agent often writes "AWS Lambda" or "Amazon Lambda"; ground truth
+ just says "Lambda". They should match either way."""
+ ok, _ = check_services_affected(["AWS Lambda"], ["Lambda"])
+ assert ok
+ ok, _ = check_services_affected(["Amazon EC2"], ["EC2"])
+ assert ok
+ ok, _ = check_services_affected(["Lambda"], ["AWS Lambda"])
+ assert ok
+
+
+def test_services_no_overlap_fails():
+ ok, msg = check_services_affected(["EC2"], ["Lambda"])
+ assert not ok
+ assert "no service overlap" in msg
+
+
+def test_services_partial_overlap_passes():
+ """At least ONE expected service must be present — pinning all services
+ would be too strict (the agent often surfaces extras)."""
+ ok, _ = check_services_affected(["Lambda", "DynamoDB"], ["Lambda"])
+ assert ok
+
+
+def test_evidence_keywords_case_insensitive_substring():
+ evidence = ["KeyError on 'user_id'", "8/8 invocations failed"]
+ ok, _ = check_evidence_keywords(evidence, ["keyerror", "user_id"])
+ assert ok
+
+
+def test_evidence_keywords_partial_meets_threshold():
+ """Default ratio is 0.5 — half the keywords is enough."""
+ evidence = ["KeyError on 'user_id' raised"]
+ ok, msg = check_evidence_keywords(evidence, ["KeyError", "user_id", "fictional"])
+ assert ok # 2/3 = 0.66, above 0.5
+ assert "2/3" in msg
+
+
+def test_evidence_keywords_below_threshold_fails():
+ evidence = ["something happened"]
+ ok, msg = check_evidence_keywords(evidence, ["KeyError", "user_id", "traceback"])
+ assert not ok
+ assert "0/3" in msg
+
+
+def test_evidence_keywords_empty_expected_passes():
+ """Scenarios that don't pin keywords just skip this check."""
+ ok, _ = check_evidence_keywords(["any"], [])
+ assert ok
+
+
+def test_tools_called_any_of_passes_on_single_overlap():
+ ok, _ = check_tools_called(
+ ["describe_log_groups", "get_log_events"],
+ ["get_log_events", "query_logs_insights"],
+ )
+ assert ok
+
+
+def test_tools_called_no_overlap_fails():
+ ok, msg = check_tools_called(["get_alarms"], ["get_log_events"])
+ assert not ok
+ assert "get_log_events" in msg
+
+
+def test_tools_called_empty_expectation_passes():
+ ok, _ = check_tools_called(["whatever"], [])
+ assert ok
+
+
+# ── full score() ─────────────────────────────────────────────────────────────
+
+
+@pytest.fixture
+def lambda_crash_ground_truth():
+ return {
+ "root_cause_category": "COMPONENT_FAILURE",
+ "services_affected": ["Lambda"],
+ "evidence_keywords": ["KeyError", "user_id"],
+ "expected_tools_any_of": ["get_log_events", "query_logs_insights"],
+ }
+
+
+def test_score_passing_run(lambda_crash_ground_truth):
+ """A correctly-investigated Lambda crash. The agent identified the right
+ category, listed Lambda, quoted the traceback, and inspected logs."""
+ agent_output = {
+ "root_cause_category": "COMPONENT_FAILURE",
+ "root_cause_summary": "Lambda raises KeyError on 'user_id'",
+ "evidence": ["traceback: KeyError on 'user_id'", "8/8 invocations failed"],
+ "services_affected": ["AWS Lambda"],
+ }
+ metrics = {
+ "tools_called": ["describe_log_groups", "get_log_events"],
+ "tool_call_count": 2,
+ "latency_ms": 42_000,
+ "input_tokens": 5_000,
+ "output_tokens": 800,
+ "cost_usd": 0.018,
+ "model": "openrouter/anthropic/claude-sonnet-4-6",
+ }
+ r = score("001_lambda_crashing", agent_output, lambda_crash_ground_truth, metrics)
+ assert r.passed
+ assert r.latency_ms == 42_000
+ assert r.tool_call_count == 2
+ assert r.cost_usd == 0.018
+
+
+def test_score_fails_on_wrong_root_cause(lambda_crash_ground_truth):
+ """The agent saw the right symptom but classified it as a resource limit
+ instead of a component failure — that's wrong."""
+ agent_output = {
+ "root_cause_category": "RESOURCE_LIMIT",
+ "evidence": ["traceback: KeyError on 'user_id'"],
+ "services_affected": ["Lambda"],
+ }
+ r = score("001", agent_output, lambda_crash_ground_truth,
+ {"tools_called": ["get_log_events"]})
+ assert not r.passed
+ assert any("RESOURCE_LIMIT" in m for m in r.reasons)
+
+
+def test_score_fails_on_no_log_inspection(lambda_crash_ground_truth):
+ """The agent classified correctly but never looked at the logs — it just
+ guessed. Methodology check should fail."""
+ agent_output = {
+ "root_cause_category": "COMPONENT_FAILURE",
+ "evidence": ["KeyError on user_id"],
+ "services_affected": ["Lambda"],
+ }
+ r = score("001", agent_output, lambda_crash_ground_truth,
+ {"tools_called": ["get_alarms"]}) # never checked logs
+ assert not r.passed
+ assert any("expected tools" in m.lower() for m in r.reasons)
+
+
+def test_score_fails_on_generic_evidence(lambda_crash_ground_truth):
+ """The agent classified the category right but its evidence is generic
+ boilerplate that could describe any failure — keyword check fails."""
+ agent_output = {
+ "root_cause_category": "COMPONENT_FAILURE",
+ "evidence": ["the function is broken", "errors happened"],
+ "services_affected": ["Lambda"],
+ }
+ r = score("001", agent_output, lambda_crash_ground_truth,
+ {"tools_called": ["get_log_events"]})
+ assert not r.passed
+ assert any("evidence" in m.lower() for m in r.reasons)
+
+
+def test_score_services_match_falls_back_to_summary_text(lambda_crash_ground_truth):
+ """Agents frequently fill services_affected with the affected resource NAME
+ ('opendevops-demo-crashing') rather than the service CATEGORY ('Lambda').
+ When the summary mentions the service, that should be enough to match."""
+ agent_output = {
+ "root_cause_category": "COMPONENT_FAILURE",
+ "root_cause_summary": "The Lambda function opendevops-demo-crashing is failing.",
+ "evidence": ["KeyError on user_id"],
+ "services_affected": ["opendevops-demo-crashing"], # resource name, not category
+ }
+ r = score("001", agent_output, lambda_crash_ground_truth,
+ {"tools_called": ["get_log_events"]})
+ assert r.passed, f"reasons: {r.reasons}"
+
+
+def test_score_evidence_keywords_match_against_summary_too(lambda_crash_ground_truth):
+ """Some agents put diagnostic detail in root_cause_summary and leave
+ the structured evidence list sparse. The scorer should scan both."""
+ agent_output = {
+ "root_cause_category": "COMPONENT_FAILURE",
+ "root_cause_summary": "Lambda is failing with KeyError on user_id.",
+ "evidence": ["Something happened."], # the keywords aren't in the evidence list
+ "services_affected": ["Lambda"],
+ }
+ r = score("001", agent_output, lambda_crash_ground_truth,
+ {"tools_called": ["get_log_events"]})
+ assert r.passed, f"reasons: {r.reasons}"
+
+
+def test_score_records_soft_metrics_even_when_failing(lambda_crash_ground_truth):
+ """A failure shouldn't lose the cost/latency numbers — they're how we
+ track regressions even when accuracy stays at 100%."""
+ r = score(
+ "001",
+ {"root_cause_category": "UNKNOWN", "evidence": [], "services_affected": []},
+ lambda_crash_ground_truth,
+ {
+ "tools_called": [],
+ "latency_ms": 19_000,
+ "input_tokens": 1234,
+ "output_tokens": 56,
+ "cost_usd": 0.0012,
+ "model": "openrouter/x",
+ },
+ )
+ assert not r.passed
+ assert r.latency_ms == 19_000
+ assert r.input_tokens == 1234
+ assert r.model == "openrouter/x"
diff --git a/apps/documentation/benchmark.md b/apps/documentation/benchmark.md
new file mode 100644
index 0000000..0f36430
--- /dev/null
+++ b/apps/documentation/benchmark.md
@@ -0,0 +1,179 @@
+# Benchmark
+
+Reproducible results for the OpenDevOps investigation agent on the bundled
+incident benchmark. Every number here is regenerable with `make eval` (see
+[`eval.md`](./eval.md) for the harness). This page is the "show me it works"
+artifact — for engineers evaluating the tool and for sharing the headline
+numbers.
+
+> **Reproducibility is the point.** These aren't cherry-picked transcripts.
+> Each scenario stands up a real broken AWS/Azure resource, the agent
+> investigates over the same `/chat` API a user hits, and the answer is scored
+> against ground truth. Re-run it yourself and you should land within noise of
+> these figures.
+
+---
+
+## Headline KPIs
+
+Latest sweep — **2026-06-02**, model `openrouter/openai/gpt-oss-120b` (a
+commodity open-weight model, deliberately *not* a frontier model):
+
+| Metric | Result |
+|---|---|
+| **Root-cause accuracy** | **9 / 10 (90%)** |
+| **Median time to root cause** | **~52 s** |
+| Total time, 10 incidents | ~9 min (541.6 s) |
+| **Cost per investigation** | **~$0.03** (≈$0.25–0.40 for all 10)¹ |
+| Tool calls per investigation | ~9 |
+| Cloud APIs touched | CloudWatch, CloudTrail, Lambda, ECS, IAM, EC2, Azure VM |
+
+¹ In-app token accounting is being calibrated (it currently under-counts
+reasoning tokens — tracked in issue #59). The cost figures here use the
+**OpenRouter dashboard actuals** as the source of truth, expressed as a range.
+
+**The story in one line:** *even on a cheap open model, the agent finds the
+true root cause of 9 out of 10 real incidents in under a minute each, for a few
+cents.*
+
+---
+
+## Why these three KPIs
+
+| KPI | Who it's for | Why it matters |
+|---|---|---|
+| **Accuracy (9/10)** | Engineers | Trust. A faster wrong answer is worthless on call. 90% on a commodity model — frontier models score higher (see below). |
+| **Time to root cause (~52 s)** | Engineers / SRE leads | MTTR. The agent correlates alarms, logs, metrics and CloudTrail in ~1 min — work that takes an on-call engineer tens of minutes of manual digging. |
+| **Cost per investigation (~$0.03)** | Buyers / investors | Unit economics. The marginal cost of an investigation is cents, and it's model-portable — it drops as open models get cheaper. |
+
+---
+
+## Full results
+
+| # | Scenario | Result | Latency | Tools | Root cause found |
+|---|---|---|---|---|---|
+| 001 | Lambda crashing (`KeyError`) | ✅ PASS | 50.9 s | 9 | SYSTEM_CHANGE |
+| 002 | Lambda throttled (concurrency) | ✅ PASS | 53.1 s | 12 | RESOURCE_LIMIT |
+| 003 | Azure VM deallocated | ✅ PASS | 52.1 s | 9 | SYSTEM_CHANGE |
+| 004 | CloudWatch alarm → failing Lambda | ✅ PASS | ~52 s | ~10 | COMPONENT_FAILURE |
+| 005 | Lambda → DynamoDB throttle | ✅ PASS | 100.9 s | 12 | RESOURCE_LIMIT |
+| 006 | IAM permission regression | ✅ PASS | 23.7 s | 5 | SYSTEM_CHANGE |
+| 007 | Lambda VPC no egress | ❌ FAIL | 62.0 s | 10 | (mis-categorized²) |
+| 008 | Bad deploy (`JSONDecodeError`) | ✅ PASS | 42.2 s | 6 | SYSTEM_CHANGE |
+| 009 | Lambda timeout misconfig | ✅ PASS | 64.1 s | 9 | RESOURCE_LIMIT |
+| 010 | ECS image pull failure | ✅ PASS | 41.1 s | 9 | SYSTEM_CHANGE |
+
+² **The one honest miss.** On the VPC-no-egress incident the agent's *narrative*
+was correct ("Lambda runs in a VPC without a NAT gateway, lacks outbound
+connectivity") but it labeled the category `RESOURCE_LIMIT` instead of a
+routing/dependency cause — contradicting its own evidence. We deliberately do
+**not** loosen the scorer to accept it; a benchmark that excuses its model's
+mistakes is worthless. Frontier models (e.g. Claude Sonnet, Gemini 2.5) classify
+this one correctly.
+
+---
+
+## Cost & speed comparison
+
+> ⚠️ **Verify the competitor rate before quoting this externally.** The
+> per-second figure below is an assumption, not a sourced number — confirm it
+> against the vendor's published pricing before putting it in front of
+> customers or investors.
+
+Framing: *if a per-second-billed managed agent took the same wall-clock time to
+work these 10 incidents, what would it cost vs OpenDevOps?*
+
+| | Time (10 incidents) | Cost | Per incident |
+|---|---|---|---|
+| Per-second managed agent @ ~$0.008/s *(assumed)* | 541.6 s | **~$4.33** | ~$0.43 |
+| **OpenDevOps** (gpt-oss-120b via OpenRouter) | 541.6 s | **~$0.25–0.40** | **~$0.03** |
+| | | **≈ 10–15× cheaper** | |
+
+This is a like-for-like *wall-clock* comparison; it assumes the competitor
+takes the same time, which should itself be measured rather than assumed. The
+durable point isn't the exact multiple — it's the **structural** one: OpenDevOps'
+marginal cost is the LLM token bill, which is a few cents on commodity models
+and keeps falling, with no per-seat or per-second platform tax.
+
+---
+
+## ROI: what one investigation actually replaces
+
+> Assumptions are explicit and conservative — adjust the engineer rate to your
+> own loaded cost. These are illustrative unit economics, not measured in this
+> benchmark.
+
+The real comparison isn't agent-vs-agent — it's **agent-vs-an-engineer doing it
+by hand**. Working one of these incidents manually means an on-call engineer
+pulling CloudWatch alarms, log groups, metric graphs, and CloudTrail, then
+correlating them.
+
+| | Per incident |
+|---|---|
+| Manual triage by an engineer (20–40 min @ ~$100/hr loaded) | **~$33–67** |
+| OpenDevOps investigation (~52 s, gpt-oss-120b) | **~$0.03** |
+| **Return on the LLM spend** | **~1,000–2,000×** |
+
+Put differently: the agent replaces ~$50 of skilled engineer time with ~3 cents
+of compute — and returns the answer in under a minute instead of half an hour.
+
+### Monthly run-rate at scale
+
+A team handling **500 incidents/month**:
+
+| | Monthly cost |
+|---|---|
+| Engineer triage time (500 × ~$50) | **~$25,000** |
+| Per-second managed agent (500 × ~$0.43) | **~$215** |
+| **OpenDevOps** (500 × ~$0.03) | **~$15** |
+
+The headline isn't "$15 vs $215" — it's that **$15 of compute deflects ~$25,000
+of toil**, and that ratio holds (or improves) as open-model prices fall.
+
+---
+
+## Model portability (the real moat)
+
+The harness scores any model behind the same `/chat` API — swap the model in
+**Settings → Agent config → LLM** and re-run. Observed pattern:
+
+- **Frontier models** (Claude Sonnet/Opus, Gemini 2.5 Pro) — highest accuracy,
+ reliable tool-calling. Use for production where being right matters most.
+- **Mid-tier / open models** (gpt-oss-120b, Gemini 2.5 Flash) — ~8–9/10 at a
+ fraction of the cost. Strong default for cost-sensitive deployments.
+- **Small models** (sub-70B, budget tiers) — fall off, mostly by failing to
+ emit a structured conclusion rather than by reasoning errors.
+
+You are never locked to one vendor's model or price. As open models improve,
+the same agent gets better and cheaper for free.
+
+---
+
+## Caveats (read these before quoting numbers)
+
+1. **Cost accuracy.** In-app token accounting under-counts reasoning tokens for
+ some providers (issue #59). Treat the OpenRouter/Anthropic dashboard as the
+ billing source of truth until that's reconciled.
+2. **Competitor pricing is unverified** — see the warning above.
+3. **MTTR vs humans is illustrative**, not measured here. The ~52 s is the
+ agent's; the "tens of minutes" human baseline is an industry-typical range,
+ not a controlled A/B in this benchmark.
+4. **Sample size is 10 scenarios.** Enough to catch regressions and set a
+ directional headline; not a statistical claim. Expanding the suite is
+ ongoing.
+5. **Numbers are model- and time-stamped.** Re-running on a different model or
+ date will move them — that's the intended use, not a bug.
+
+---
+
+## Reproduce it
+
+```bash
+# one full sweep (real cloud + real LLM cost — see eval.md cost guard)
+make eval # or:
+uv run python demos/eval/run.py --setup-profile default
+```
+
+Results land in `demos/eval/results//report.md` (+ `results.json`
+for dashboards). Re-run on your own model to generate your own row in the
+portability table above.
diff --git a/apps/documentation/eval.md b/apps/documentation/eval.md
new file mode 100644
index 0000000..1ccc5ff
--- /dev/null
+++ b/apps/documentation/eval.md
@@ -0,0 +1,192 @@
+# Eval
+
+A small reproducible benchmark for the investigation agent. Each scenario stands
+up a known-bad AWS or Azure state, asks the agent to investigate, then scores the
+agent's `submit_investigation` output against ground truth. Use it to:
+
+- Get the README headline number (*"X/N root causes found, $Y/run, Z s median"*).
+- Catch regressions when you upgrade models, change prompts, or refactor tools.
+- Decide empirically which LLM is reliable enough for production use.
+
+> Looking for the published results rather than how to run them? See
+> [`benchmark.md`](./benchmark.md).
+
+> This is **integration** code — it hits real AWS / Azure and pays real LLM
+> costs. It's deliberately *not* a pytest target. Run it nightly in a dedicated
+> CI lane, or manually before releases.
+
+---
+
+## Quick start
+
+```bash
+# Prereqs:
+# 1. Agent dev server running (e.g. `make dev` -> http://localhost:8000).
+# 2. An LLM provider configured in apps/backend/.env (any of OPENROUTER_API_KEY,
+# ANTHROPIC_API_KEY, OPENAI_API_KEY, GROQ_API_KEY, GEMINI_API_KEY, or a
+# logged-in Claude Code CLI).
+# 3. AWS creds with permission to create the scenario's resources (Lambda,
+# throwaway IAM role). Azure scenarios need a logged-in `az` session.
+
+uv run python demos/eval/run.py # all scenarios
+uv run python demos/eval/run.py --scenario 001 # one (by id prefix)
+uv run python demos/eval/run.py --setup-profile default # see below
+```
+
+Each run writes to `demos/eval/results//`:
+
+- `report.md` — pass/fail summary, per-scenario detail, soft metrics
+ (latency, cost, tokens, tool-call count).
+- `results.json` — same data, machine-readable for CI dashboards.
+- One log per `setup` / `teardown` subprocess so any cloud error
+ (`AccessDeniedException`, etc.) is preserved verbatim even if the terminal
+ truncates it.
+
+---
+
+## How it works
+
+```
+ ┌──────────────┐
+demos/{aws,azure}/