feat(eval): reproducible incident benchmark for the agent

.gitignore

@@ -41,3 +41,6 @@ Thumbs.db
 
 # SQLite data directory (CHECKPOINT_BACKEND=sqlite)
 data/
+
+# Eval per-run output — per-machine artifacts, never check in
+demos/eval/results/

Makefile

@@ -1,4 +1,4 @@
-.PHONY: dev ui migrate test lint frontend-dev frontend-build help
+.PHONY: dev ui migrate test lint frontend-dev frontend-build eval help
 
 BACKEND := apps/backend
 FRONTEND := apps/frontend
@@ -32,6 +32,12 @@ frontend-dev:
 frontend-build:
 	cd $(FRONTEND) && npm install && npm run build
 
+# ── Eval (integration; real cloud + real LLM spend) ────────────────────────────
+# Needs the agent dev server running (make dev) and AWS / Azure creds for the
+# scenarios you're running. See demos/eval/README.md.
+eval:
+	python demos/eval/run.py
+
 # ── Full stack (Docker Compose) ────────────────────────────────────────────────
 compose-up:
 	docker compose -f deployment/docker-compose/docker-compose.yml up --build
@@ -43,4 +49,5 @@ compose-down:
 help:
 	@echo "Backend:   make dev | ui | migrate | test | lint | lint-fix | install"
 	@echo "Frontend:  make frontend-dev | frontend-build"
+	@echo "Eval:      make eval     (integration — hits real cloud + LLM)"
 	@echo "Docker:    make compose-up | compose-down"

README.md

@@ -5,6 +5,17 @@ Anthropic, OpenRouter, Groq, Gemini, Mistral, **Ollama for air-gapped / regulate
 reuse your existing **Claude Code** subscription (auto-detected). Investigates incidents, finds root
 causes, and gives actionable mitigation plans — without the cloud-vendor DevOps-agent price tag.
 
+> ### 📊 Benchmarked, not just demoed
+> On a **reproducible 10-incident suite** (real AWS + Azure resources, scored against ground truth), running on a **commodity open model** (`gpt-oss-120b` — no frontier model required):
+>
+> | Root causes found | Median time | Cost / investigation | vs. AWS DevOps Agent | vs. manual triage |
+> |:---:|:---:|:---:|:---:|:---:|
+> | **9 / 10 (90%)** | **~52 s** | **~$0.03** | **~10× cheaper**¹<br/>(~$0.03 vs ~$0.43) | **~1,000× cheaper**² |
+>
+> *~$0.03 of compute replaces ~$50 of engineer toil — and costs a fraction of a managed cloud DevOps agent — while returning the answer in under a minute instead of half an hour.* Reproduce it with `make eval` → **[full benchmark & methodology](apps/documentation/benchmark.md)**.
+>
+> <sub>¹ vs. AWS DevOps Agent — its per-second rate applied to the same wall-clock time (~$0.43/investigation); verify against AWS's published pricing. ² Illustrative unit economics vs. ~20–40 min of on-call triage. Cost shown is the provider-dashboard actual; see [caveats](apps/documentation/benchmark.md#caveats-read-these-before-quoting-numbers).</sub>
+
 **Cloud setup:** [AWS (IAM)](apps/documentation/iam_setup.md) · [Azure (service principal / login)](apps/documentation/azure_setup.md)
 
 ## Demo

apps/backend/src/api/routers/chat.py

@@ -60,6 +60,35 @@ def _clean(text: str) -> str:
     return _CHANNEL_RE.sub("", text)
 
 
+def _extract_text(content: Any) -> str:
+    """Pull the user-visible text out of a chunk's ``content``, regardless of shape.
+
+    Reasoning / hybrid models (gpt-oss, OpenAI o1-family, some Anthropic configs,
+    Gemini Thinking) return ``content`` as a list of typed blocks:
+
+        [{'type': 'thinking', 'thinking': '...internal reasoning...'},
+         {'type': 'text',     'text':     'the actual answer'}]
+
+    Only the ``text`` blocks go to the UI; ``thinking`` blocks are billed but
+    not shown (they're shown via the tool-call inspector / reasoning UI if we
+    ever surface them). Plain-string content (older models, Sonnet, GPT-4o)
+    flows through unchanged.
+    """
+    if not content:
+        return ""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts: list[str] = []
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "text":
+                txt = block.get("text") or ""
+                if isinstance(txt, str):
+                    parts.append(txt)
+        return "".join(parts)
+    return ""
+
+
 
 
 async def _stream_chat(session_id: str, user_message: str):
@@ -118,6 +147,13 @@ def _flush_text_buf():
 
     await maybe_summarize(agent, config, session_id)
 
+    # DEBUG_LLM_CHUNKS=true in .env dumps every raw chunk's content + tool calls +
+    # usage to the log. Useful when a model returns "(no response)" in the UI to
+    # see whether the content is being stripped, wrapped in an unexpected format
+    # (e.g. gpt-oss harmony channels), or is genuinely empty from the provider.
+    import os as _os
+    _debug_chunks = _os.environ.get("DEBUG_LLM_CHUNKS", "").lower() in ("1", "true", "yes")
+
     try:
         async with asyncio.timeout(settings.investigation_timeout):
             async for chunk, _meta in agent.astream(
@@ -129,6 +165,20 @@ def _flush_text_buf():
                     logger.info("[{sid}]  cancelled by user", sid=sid)
                     yield f"data: {json.dumps({'type': 'cancelled'})}\n\n"
                     return
+                if _debug_chunks:
+                    _raw = getattr(chunk, "content", None)
+                    _raw_type = type(_raw).__name__
+                    _raw_preview = repr(_raw)[:400] if _raw is not None else "None"
+                    _tcc = getattr(chunk, "tool_call_chunks", None)
+                    _tc = getattr(chunk, "tool_calls", None)
+                    _um = getattr(chunk, "usage_metadata", None)
+                    logger.info(
+                        "[{sid}] RAW chunk type={t} content={c}  tcc={tcc}  tc={tc}  usage={u}",
+                        sid=sid, t=_raw_type, c=_raw_preview,
+                        tcc=repr(_tcc)[:200] if _tcc else "-",
+                        tc=repr(_tc)[:200] if _tc else "-",
+                        u=repr(_um)[:200] if _um else "-",
+                    )
                 um = getattr(chunk, "usage_metadata", None)
                 if um:
                     usage_meta = um
@@ -160,9 +210,10 @@ def _flush_text_buf():
                         }
 
                 content = getattr(chunk, "content", "")
-                if content and isinstance(content, str):
+                visible = _extract_text(content)
+                if visible:
                     if not getattr(chunk, "tool_call_id", None):
-                        text_buf += content
+                        text_buf += visible
                         new_clean = _clean(text_buf)
                         delta = new_clean[len(clean_buf):]
                         if delta:
@@ -199,13 +250,22 @@ def _flush_text_buf():
                     )
                     if isinstance(result, dict) and "error" in result:
                         logger.warning("   [{sid}]  ⚠   error: {err}", sid=sid, err=result["error"])
-                    else:
-                        keys  = list(result.keys()) if isinstance(result, dict) else type(result).__name__
+                    elif isinstance(result, dict):
+                        keys  = list(result.keys())
                         count = result.get("count", result.get("events", result.get("functions", "")))
                         logger.info(
                             "   [{sid}]  ✓   keys={keys}  count={count}",
                             sid=sid, keys=keys, count=count if isinstance(count, int) else "—",
                         )
+                    else:
+                        # gpt-oss / some providers return tool results as JSON arrays
+                        # (e.g. CloudWatch metric data) — preserve them but don't try
+                        # to .get() into a list. Log shape only.
+                        logger.info(
+                            "   [{sid}]  ✓   type={t}  len={n}",
+                            sid=sid, t=type(result).__name__,
+                            n=len(result) if hasattr(result, "__len__") else "—",
+                        )
 
                     tool_calls_log.append({"tool": call_info["tool"], "args": call_info["args"], "result": result})
                     label = _pick_label(call_info["tool"])
@@ -243,6 +303,17 @@ def _flush_text_buf():
     if usage_meta:
         usage["input_tokens"]  = _field(usage_meta, "input_tokens", 0) or 0
         usage["output_tokens"] = _field(usage_meta, "output_tokens", 0) or 0
+        # Capture reasoning tokens (gpt-oss, o1-family, Gemini thinking) for visibility
+        # in the cost card and downstream analysis. We intentionally do NOT add them
+        # to output_tokens for the cost calc here: providers differ on whether
+        # output_tokens already includes reasoning (gpt-oss: YES — 53 total includes
+        # 38 reasoning; Gemini: NO — but reasoning is also missing from the metadata,
+        # which is the actual undercount bug). Per-provider normalization is tracked
+        # in issue #59.
+        details = _field(usage_meta, "output_token_details", None) or {}
+        reasoning = _field(details, "reasoning", 0) or 0
+        if reasoning and reasoning > 0:
+            usage["reasoning_tokens"] = reasoning
     cost = calc_cost(
         usage["model"],
         usage.get("input_tokens", 0),

apps/backend/tests/test_eval/test_salvage.py

@@ -0,0 +1,63 @@
+"""Unit tests for the runner's final-message salvage (demos/eval/run.py).
+
+The runner is integration-only, but ``_salvage_submit_json`` is a pure parser
+and the single thing standing between "model gave the right answer" and a FAIL
+when a model (notably gpt-oss) emits the ``submit_investigation`` payload as
+JSON text instead of a tool call. Pin its behavior so a contributor editing the
+brace-matching logic sees a clear failure.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+_EVAL_DIR = Path(__file__).resolve().parents[4] / "demos" / "eval"
+sys.path.insert(0, str(_EVAL_DIR))
+
+from run import _salvage_submit_json  # noqa: E402
+
+
+def test_plain_json_dump_is_recovered():
+    """The common gpt-oss case: the whole message is the submit payload."""
+    payload = {
+        "root_cause_category": "SYSTEM_CHANGE",
+        "root_cause_summary": "bad image tag",
+        "evidence": ["ECS service reports desired=1, running=0"],
+    }
+    got = _salvage_submit_json(json.dumps(payload))
+    assert got == payload
+
+
+def test_json_wrapped_in_prose_and_fences_is_recovered():
+    """Some models wrap the JSON in explanatory prose and/or ``` fences."""
+    payload = {"root_cause_category": "RESOURCE_LIMIT", "evidence": ["x"]}
+    msg = f"Here is my analysis.\n```json\n{json.dumps(payload)}\n```\nDone."
+    assert _salvage_submit_json(msg) == payload
+
+
+def test_braces_and_quotes_inside_string_values_dont_break_matching():
+    """A traceback in the summary contains both quotes and braces — the
+    string-aware brace matcher must not miscount depth on those."""
+    payload = {
+        "root_cause_summary": 'KeyError at user = event["user_id"] with {braces}',
+        "root_cause_category": "COMPONENT_FAILURE",
+        "evidence": ["a", "b"],
+    }
+    msg = "Some prose. " + json.dumps(payload) + " trailing words."
+    assert _salvage_submit_json(msg) == payload
+
+
+def test_prose_without_schema_returns_empty():
+    """A fallback message ('see tool calls') must NOT be mistaken for an answer."""
+    assert _salvage_submit_json("Investigation complete. See tool calls for details.") == {}
+
+
+def test_non_schema_json_object_is_ignored():
+    """A JSON object that lacks the investigation keys is not a submit payload."""
+    assert _salvage_submit_json('{"foo": 1, "bar": [2, 3]}') == {}
+
+
+def test_empty_input_returns_empty():
+    assert _salvage_submit_json("") == {}