feat(evidence): replayable evidence pack and ranked-hypotheses schema

CHANGELOG.md

@@ -30,6 +30,21 @@ Notable changes to OpenDevOps Agent (open-source core + backend).
   (`None` = unscoped — OSS behavior unchanged).
 - `CREDENTIALS_ENCRYPTION_KEY` (Fernet) for encrypting stored account secrets.
 
+### Added — Replayable evidence pack & ranked hypotheses
+- **Evidence pack endpoint** — read-only `GET /api/sessions/{id}/evidence` returns the
+  investigation's ranked hypotheses, each with cited evidence linked to the supporting tool
+  call, the exact query/command that ran, and a deterministic AWS-console deeplink. Reads the
+  conclusion from the persisted `submit_investigation` tool call (the `findings` table stays
+  an unwritten placeholder). Pure builder + console-deeplink encoder live in
+  `opendevops_core/agent/evidence.py`; `db.get_evidence()` added to the `DatabaseBackend` ABC
+  (default + all three backends). Frontend `EvidencePanel` renders grouped hypotheses + replay
+  cards with copy-to-clipboard and JSON export. See `apps/documentation/evidence_pack.md`.
+- **Ranked hypotheses conclusion schema** — `submit_investigation` now emits
+  `hypotheses: list[dict]` (`{hypothesis, evidence, confidence}`) alongside the legacy
+  `root_cause_summary` + flat `evidence[]` (preserved for backward compatibility). Migration
+  `015` adds `findings.hypotheses JSONB` (postgres only). The builder falls back to one
+  synthetic hypothesis when `hypotheses` is absent so pre-existing investigations still render.
+
 ### Changed
 - **README** reframed as multi-cloud (AWS + Azure) with links to both setup guides.
 - **`demos/`** reorganized into `demos/aws/` + `demos/azure/` with a top-level index.

CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

README.md

@@ -65,6 +65,7 @@ On a **reproducible 10-incident suite** (real AWS + Azure resources, scored agai
 - **AWS Configuration settings tab** — admin-only editable tab in Settings for SQS Queue URL and AWS Region; shared org-wide via database-backed app config; includes an inline IAM permission checker per service
 - **Web UI** — React + Vite SPA served by FastAPI:
   - **Chat page** — streaming responses, collapsible tool call inspector, cost/latency card, stop button; supports `?prompt=` deeplink for pre-seeded investigations from the Monitoring dashboard
+    - **Replayable evidence pack** — an Evidence button opens the investigation's ranked hypotheses, each with cited evidence linked to the supporting tool call, the exact query/command that ran, and a deterministic AWS-console deeplink; copy-to-clipboard and JSON export. Served read-only from `GET /api/sessions/{id}/evidence` — see [apps/documentation/evidence_pack.md](apps/documentation/evidence_pack.md)
   - **Session history sidebar** — lists all past conversations; click any to resume with full tool call inspector and cost card restored; new chat and delete (soft) buttons
   - **Monitoring page** — live incident feed from event-driven detection; alert detail with investigate deeplink
   - **Dashboard** — session counts, tool call stats, cost/latency, context saved, activity chart, service breakdown, root cause distribution, recent sessions

apps/backend/src/api/app.py

@@ -20,6 +20,7 @@
     auth,
     chat,
     dashboard,
+    evidence,
     history,
     integrations,
     monitoring,
@@ -185,6 +186,7 @@ async def lifespan(_app: FastAPI):
 
 app.include_router(chat.router)
 app.include_router(sessions.router)
+app.include_router(evidence.router)
 app.include_router(dashboard.router)
 app.include_router(history.router)
 app.include_router(auth.router)

apps/backend/src/api/routers/evidence.py

@@ -0,0 +1,20 @@
+"""Replayable evidence pack — read-only view over a session's persisted tool calls.
+
+Joins the investigation conclusion's hypotheses to the supporting tool calls that produced
+them, surfacing the exact query/command that ran plus a deterministic console deeplink.
+Uses the `/api/sessions` prefix so the SPA fallback never intercepts it.
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter
+from opendevops_core.agent.db import db
+from opendevops_core.agent.evidence import build_evidence_pack
+
+router = APIRouter(prefix="/api/sessions", tags=["evidence"])
+
+
+@router.get("/{session_id}/evidence")
+async def get_evidence(session_id: str) -> dict:
+    raw = await db.get_evidence(session_id)
+    return build_evidence_pack(session_id, raw["aws_region"], raw["tool_calls"])

apps/backend/tests/test_api/test_evidence.py

@@ -0,0 +1,152 @@
+"""Tests for GET /api/sessions/{id}/evidence — the replayable evidence pack endpoint."""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+os.environ.setdefault("CHECKPOINT_BACKEND", "memory")
+os.environ.setdefault("LLM_MODEL", "openrouter/anthropic/claude-3.5-sonnet")
+os.environ.setdefault("LLM_API_KEY", "test-key")
+
+
+async def _seed_investigation(session_id: str) -> None:
+    """Persist a session with supporting tool calls + a submit_investigation conclusion."""
+    from opendevops_core.agent.db import db
+
+    await db.upsert_session(session_id, "test-model", "us-east-1", title="Lambda throttling")
+    msg_id = await db.save_message(session_id, "assistant", "Investigation complete.")
+
+    await db.save_tool_call(
+        session_id,
+        msg_id,
+        "get_metric_data",
+        {
+            "namespace": "AWS/Lambda",
+            "metric": "Throttles",
+            "dimensions": [{"Name": "FunctionName", "Value": "payment-fn"}],
+        },
+        {"count": 1, "datapoints": [{"timestamp": "t", "value": 120}]},
+    )
+    await db.save_tool_call(
+        session_id,
+        msg_id,
+        "query_logs_insights",
+        {
+            "log_group": "/aws/lambda/payment-fn",
+            "query": "fields @timestamp, @message | filter @message like /Throttl/",
+        },
+        {"results": []},
+    )
+    await db.save_tool_call(
+        session_id,
+        msg_id,
+        "run_bash_command",
+        {"command": "az monitor metrics list --resource payment-fn"},
+        {"stdout": "ok"},
+    )
+    await db.save_tool_call(
+        session_id,
+        msg_id,
+        "submit_investigation",
+        {
+            "root_cause_category": "RESOURCE_LIMIT",
+            "root_cause_summary": "payment-fn hit its concurrency limit",
+            "hypotheses": [
+                {
+                    "hypothesis": "Concurrency limit reached on payment-fn",
+                    "evidence": ["Throttles metric on payment-fn spiked to 120"],
+                    "confidence": "HIGH",
+                },
+                {
+                    "hypothesis": "Downstream dependency slow",
+                    "evidence": ["No corroborating evidence found"],
+                    "confidence": "LOW",
+                },
+            ],
+            "evidence": ["Throttles metric on payment-fn spiked to 120"],
+            "confidence": "HIGH",
+        },
+        {},
+    )
+
+
+@pytest.mark.asyncio
+async def test_evidence_grouped_per_hypothesis_and_linked():
+    from httpx import ASGITransport, AsyncClient
+
+    from api.app import app
+
+    session_id = "evid-test-grouped-1"
+    await _seed_investigation(session_id)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        r = await client.get(f"/api/sessions/{session_id}/evidence")
+
+    assert r.status_code == 200
+    pack = r.json()
+
+    assert pack["has_conclusion"] is True
+    assert pack["aws_region"] == "us-east-1"
+    assert pack["root_cause_category"] == "RESOURCE_LIMIT"
+
+    # Two ranked hypotheses, most likely first.
+    assert [h["hypothesis"] for h in pack["hypotheses"]] == [
+        "Concurrency limit reached on payment-fn",
+        "Downstream dependency slow",
+    ]
+
+    # submit_investigation is the conclusion, never a replay entry.
+    assert all(tc["tool"] != "submit_investigation" for tc in pack["tool_calls"])
+    assert len(pack["tool_calls"]) == 3
+
+    # The top hypothesis's evidence links to the get_metric_data call that produced it.
+    top_ev = pack["hypotheses"][0]["evidence"][0]
+    linked_id = top_ev["tool_call_id"]
+    assert linked_id is not None
+    linked = next(tc for tc in pack["tool_calls"] if tc["id"] == linked_id)
+    assert linked["tool"] == "get_metric_data"
+
+
+@pytest.mark.asyncio
+async def test_evidence_exposes_command_and_console_deeplink():
+    from httpx import ASGITransport, AsyncClient
+
+    from api.app import app
+
+    session_id = "evid-test-deeplink-2"
+    await _seed_investigation(session_id)
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        r = await client.get(f"/api/sessions/{session_id}/evidence")
+
+    pack = r.json()
+    by_tool = {tc["tool"]: tc for tc in pack["tool_calls"]}
+
+    # Logs Insights: exact query verbatim + a deterministic console deeplink.
+    insights = by_tool["query_logs_insights"]
+    assert insights["command"] == "fields @timestamp, @message | filter @message like /Throttl/"
+    assert insights["console_url"].startswith("https://us-east-1.console.aws.amazon.com/cloudwatch")
+    assert "logs-insights" in insights["console_url"]
+
+    # Azure / bash: the literal command is surfaced, no console deeplink.
+    bash = by_tool["run_bash_command"]
+    assert bash["command"] == "az monitor metrics list --resource payment-fn"
+    assert bash["console_url"] is None
+
+
+@pytest.mark.asyncio
+async def test_evidence_unknown_session_is_empty():
+    from httpx import ASGITransport, AsyncClient
+
+    from api.app import app
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+        r = await client.get("/api/sessions/no-such-session/evidence")
+
+    assert r.status_code == 200
+    pack = r.json()
+    assert pack["has_conclusion"] is False
+    assert pack["hypotheses"] == []
+    assert pack["tool_calls"] == []

apps/backend/tests/test_tools/test_evidence_pack.py

@@ -0,0 +1,111 @@
+"""Unit tests for the evidence-pack builder, console deeplinks, and the ranked-hypotheses
+schema extension to submit_investigation."""
+
+from __future__ import annotations
+
+import inspect
+
+from opendevops_core.agent.evidence import (
+    build_evidence_pack,
+    console_deeplink,
+    exact_command,
+)
+
+
+def test_console_deeplink_log_group_encoding():
+    url = console_deeplink("get_log_events", {"log_group": "/aws/lambda/fn"}, "us-east-1")
+    assert url == (
+        "https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1"
+        "#logsV2:log-groups/log-group/$252Faws$252Flambda$252Ffn"
+    )
+
+
+def test_console_deeplink_logs_insights_query_detail():
+    url = console_deeplink(
+        "query_logs_insights",
+        {"log_group": "/aws/lambda/fn", "query": "fields @timestamp", "hours": 2},
+        "eu-west-1",
+    )
+    assert url.startswith(
+        "https://eu-west-1.console.aws.amazon.com/cloudwatch/home?region=eu-west-1"
+        "#logsV2:logs-insights$3FqueryDetail$3D"
+    )
+    # Relative window encoded as seconds; query string escaped (space -> *20, @ -> *40).
+    assert "start~-7200" in url
+    assert "editorString~'fields*20*40timestamp" in url
+    assert "source~(~'*2faws*2flambda*2ffn)" in url
+
+
+def test_console_deeplink_none_without_region():
+    assert console_deeplink("get_log_events", {"log_group": "/x"}, None) is None
+
+
+def test_exact_command_only_for_query_and_bash():
+    assert exact_command("query_logs_insights", {"query": "stats count(*)"}) == "stats count(*)"
+    assert exact_command("run_bash_command", {"command": "az vm list"}) == "az vm list"
+    assert exact_command("get_alarms", {"state": "ALARM"}) is None
+
+
+def _conclusion(hypotheses=None, evidence=None):
+    args = {
+        "root_cause_category": "RESOURCE_LIMIT",
+        "root_cause_summary": "throttled",
+        "confidence": "HIGH",
+        "evidence": evidence if evidence is not None else ["flat evidence"],
+    }
+    if hypotheses is not None:
+        args["hypotheses"] = hypotheses
+    return {"tool_name": "submit_investigation", "args": args, "id": "concl"}
+
+
+def test_build_pack_links_evidence_to_tool_call():
+    tool_calls = [
+        {
+            "id": "tc-metric",
+            "tool_name": "get_metric_data",
+            "args": {
+                "namespace": "AWS/Lambda",
+                "metric": "Throttles",
+                "dimensions": [{"Name": "FunctionName", "Value": "payment-fn"}],
+            },
+            "result": {"count": 1},
+        },
+        _conclusion(
+            hypotheses=[
+                {
+                    "hypothesis": "concurrency",
+                    "evidence": ["payment-fn throttled hard"],
+                    "confidence": "HIGH",
+                }
+            ]
+        ),
+    ]
+    pack = build_evidence_pack("s1", "us-east-1", tool_calls)
+
+    assert pack["has_conclusion"] is True
+    assert len(pack["tool_calls"]) == 1  # conclusion excluded from replay
+    linked = pack["hypotheses"][0]["evidence"][0]["tool_call_id"]
+    assert linked == "tc-metric"
+
+
+def test_build_pack_falls_back_to_flat_evidence_for_legacy():
+    """Old investigations without `hypotheses` still produce one grouped hypothesis."""
+    pack = build_evidence_pack("s2", "us-east-1", [_conclusion(evidence=["only flat"])])
+    assert len(pack["hypotheses"]) == 1
+    assert pack["hypotheses"][0]["evidence"][0]["text"] == "only flat"
+    assert pack["hypotheses"][0]["confidence"] == "HIGH"
+
+
+def test_build_pack_no_conclusion():
+    pack = build_evidence_pack("s3", "us-east-1", [])
+    assert pack["has_conclusion"] is False
+    assert pack["hypotheses"] == []
+
+
+def test_submit_investigation_has_hypotheses_param():
+    from opendevops_core.tools.final_answer import submit_investigation
+
+    sig = inspect.signature(submit_investigation)
+    assert "hypotheses" in sig.parameters
+    # Must stay a primitive list type so DeepAgents can infer the schema.
+    assert sig.parameters["hypotheses"].annotation == list[dict]

apps/core/src/opendevops_core/agent/db/base.py

@@ -86,6 +86,13 @@ async def get_session_model(self, session_id: str) -> str | None:
     @abstractmethod
     async def get_messages(self, session_id: str, org_id: str | None = None) -> list[dict]: ...
 
+    async def get_evidence(self, session_id: str, org_id: str | None = None) -> dict:
+        """Return the raw material for a session's evidence pack:
+        ``{"aws_region": str | None, "tool_calls": [...]}`` where each tool call carries
+        ``id``, ``tool_name``, ``args``, ``result``, ``error`` and ``created_at`` ordered
+        oldest-first. Read-only. Default returns empty — every backend overrides it."""
+        return {"aws_region": None, "tool_calls": []}
+
     @abstractmethod
     async def delete_session(self, session_id: str) -> None: ...
 

apps/core/src/opendevops_core/agent/db/memory.py

@@ -220,6 +220,25 @@ async def get_messages(self, session_id: str, org_id: str | None = None) -> list
             result.append(item)
         return result
 
+    async def get_evidence(self, session_id: str, org_id: str | None = None) -> dict:
+        session = self._sessions.get(session_id)
+        if session is None or session.get("is_deleted"):
+            return {"aws_region": None, "tool_calls": []}
+        if org_id is not None and session.get("org_id") != org_id:
+            return {"aws_region": None, "tool_calls": []}
+        tool_calls = [
+            {
+                "id": tc["id"],
+                "tool_name": tc["tool_name"],
+                "args": tc["args"],
+                "result": tc["result"],
+                "error": tc["error"],
+                "created_at": tc["created_at"],
+            }
+            for tc in self._tool_calls.get(session_id, [])
+        ]
+        return {"aws_region": session.get("aws_region"), "tool_calls": tool_calls}
+
     async def delete_session(self, session_id: str) -> None:
         if session_id in self._sessions:
             self._sessions[session_id]["is_deleted"] = True