From 1eb72c80f88c2d05c133bf4f5ee6498e1acde7b1 Mon Sep 17 00:00:00 2001 From: DanieCuevas <43822444+DanielCuevas1208@users.noreply.github.com> Date: Tue, 4 Aug 2026 00:09:02 -0700 Subject: [PATCH] feat: extend agent trace workbench --- .github/workflows/ci.yml | 4 +- CHANGELOG.md | 23 ++ README.md | 92 ++++++- agent_trace_workbench/__init__.py | 2 +- agent_trace_workbench/cli.py | 16 ++ agent_trace_workbench/export.py | 44 ++++ agent_trace_workbench/main.py | 153 ++++++++++-- agent_trace_workbench/storage.py | 34 +++ pyproject.toml | 2 +- static/styles.css | 5 + templates/dashboard.html | 29 ++- tests/test_trend_overlay.py | 396 ++++++++++++++++++++++++++++++ 12 files changed, 769 insertions(+), 31 deletions(-) create mode 100644 tests/test_trend_overlay.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0323975..671d31c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,9 +21,9 @@ jobs: python-version: ["3.11", "3.12", "3.13"] steps: - name: Check out source - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} cache: pip diff --git a/CHANGELOG.md b/CHANGELOG.md index cf35e73..3737d43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,29 @@ All notable changes to Agent Trace Workbench appear in this file. The version format follows a release cycle. A release adds one coherent capability to the workbench. +## 1.8.0 - 2026-08-04 + +### Added + +- An agent comparison overlay on the dashboard failure trend. +- A second failure line that draws one agent beside the primary series. +- A compare select on the trend filter that lists the recorded agents except the primary one. +- A legend that shows the failure rate of each drawn series. +- `GET /api/trend/overlay` route that returns both trend series for scripts. +- `GET /api/trend/overlay?format=csv` route that returns the series as a CSV attachment. +- A `series` column in the overlay CSV that marks each row as primary or compare. +- `atw trend --compare ` command that prints both failure trend series. +- `atw trend --compare --format csv` command that prints the series as CSV. +- Dashboard JSON and CSV download links that keep the compare agent and window. +- Deterministic tests for the overlay, the API routes, the CLI options, the CSV export, and the dashboard panel. + +### Changed + +- Version numbers moved to 1.8.0. +- The dashboard trend chart can now draw two failure lines on one time axis. +- The architecture now includes an agent comparison overlay beside the failure trend. +- CI now uses the latest GitHub Actions checkout and setup-python actions. + ## 1.7.0 - 2026-08-04 ### Added diff --git a/README.md b/README.md index 4fdeda4..bd39824 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ Release 1.6 adds a trend window selector and a per-day drill-down on the dashboa Release 1.7 adds a status breakdown beside the daily failure line. Each trend day shows a stacked bar of run status counts. Read the same counts from the API or the CLI. +Release 1.8 adds an agent comparison overlay to the failure trend. Choose a second agent on the dashboard and the chart draws its failure line beside the primary series. Read both series from the API, the CLI, or a CSV file. + ## Value Agent debugging needs evidence at tool boundaries. @@ -66,7 +68,7 @@ SQLite runs in WAL mode with a busy timeout. Readers keep a committed snapshot. - `models.py` defines the portable trace contract. - `handlers.py` loads local handler config and applies side-effect guards. -- `storage.py` owns the SQLite schema, WAL coordination, idempotent ingestion, and local annotations. It also computes the review list, applies bulk labels, builds the library report, computes the daily failure trend and the status breakdown, lists the runs for one day, and enforces the retention cutoff for cleanup. A cleanup log records each scheduled sweep. +- `storage.py` owns the SQLite schema, WAL coordination, idempotent ingestion, and local annotations. It also computes the review list, applies bulk labels, builds the library report, computes the daily failure trend, the status breakdown, and the agent comparison overlay, lists the runs for one day, and enforces the retention cutoff for cleanup. A cleanup log records each scheduled sweep. - `ingestion.py` watches JSON files and returns stable schema error reports. - `otlp.py` converts the OTLP JSON encoding to and from the trace contract. - `replay.py` runs guarded local handlers and records mismatches. @@ -468,6 +470,75 @@ python -m agent_trace_workbench.cli trend --statuses --agent catalog-assistant - The panel JSON and CSV links keep the active agent and window. +## Agent comparison overlay + +Compare one agent's failure line with another on the dashboard. + +Choose a second agent in the compare select above the chart. The chart draws a dashed line for that agent beside the primary line. The legend shows the failure rate of each series. + +The compare select lists every recorded agent except the primary one. It appears only when the library records two or more agents. + +Read both series over the API. + +```powershell +curl.exe "http://127.0.0.1:8000/api/trend/overlay?agent=catalog-assistant&compare=support-assistant" +``` + +The response returns the primary series and the compare series on one window. + +```json +{ + "days": 14, + "primary_agent": "catalog-assistant", + "compare_agent": "support-assistant", + "primary": [ + { + "day": "2026-07-31", + "runs": 2, + "failures": 1, + "failure_rate": 0.5 + } + ], + "compare": [ + { + "day": "2026-07-31", + "runs": 1, + "failures": 0, + "failure_rate": 0.0 + } + ] +} +``` + +Omit `agent` to compare the whole library with one agent. + +```powershell +curl.exe "http://127.0.0.1:8000/api/trend/overlay?compare=support-assistant" +``` + +Download the overlay as a CSV document. + +```powershell +curl.exe -o overlay.csv "http://127.0.0.1:8000/api/trend/overlay?compare=support-assistant&format=csv" +``` + +The file lists one row per day per series. A `series` column marks each row as primary or compare. + +```text +day,series,agent_name,runs,failures,failure_rate +2026-07-31,primary,catalog-assistant,2,1,0.5 +2026-07-31,compare,support-assistant,1,0,0.0 +``` + +Use the CLI for scripts. + +```powershell +python -m agent_trace_workbench.cli trend --agent catalog-assistant --compare support-assistant +python -m agent_trace_workbench.cli trend --compare support-assistant --format csv +``` + +The dashboard panel links to both downloads. The links keep the active agents and window. The compare agent must differ from the primary agent. + ## Saved comparisons Save a comparison for later review. @@ -1263,7 +1334,7 @@ curl.exe -X POST http://127.0.0.1:8000/api/traces ` ## Test status -The test suite covers the core flows. It covers storage, ingestion, replay, comparison, search, annotations, bulk labels, export, review, reports, retention cleanup, and scheduled cleanup. It covers the CLI, the API, collector export, the server scheduler, and the dashboard failure trend, including the agent filter, the window selector, the day drill-down, the status breakdown, and the CSV exports. +The test suite covers the core flows. It covers storage, ingestion, replay, comparison, search, annotations, bulk labels, export, review, reports, retention cleanup, and scheduled cleanup. It covers the CLI, the API, collector export, the server scheduler, and the dashboard failure trend, including the agent filter, the window selector, the day drill-down, the status breakdown, the agent comparison overlay, and the CSV exports. Run the checks with these commands. @@ -1274,7 +1345,7 @@ python scripts/check_requirements.py python -m compileall agent_trace_workbench tests ``` -Current verification passes 308 tests, Ruff lint, dependency checks, and Python compilation. CI installs from `requirements-lock.txt` and runs these checks on Python 3.11, 3.12, and 3.13 for every push and pull request. +Current verification passes 329 tests, Ruff lint, dependency checks, and Python compilation. CI installs from `requirements-lock.txt` and runs these checks on Python 3.11, 3.12, and 3.13 for every push and pull request. ## Limitations @@ -1324,6 +1395,16 @@ The trend CSV repeats the active agent in every row. The all-agents view leaves The trend export lists one row per day. It does not add a window total row. +The compare overlay matches each agent name exactly. An unknown name draws a flat line at zero. + +The compare overlay shares the primary trend window. It does not add a third series. + +The overlay CSV repeats the agent name in every row. The all-agents view leaves the primary cell empty. + +The overlay CSV lists both series in one file. Plot tools filter rows by the series column. + +The day drill-down stays bound to the primary series. It does not drill into the compare line. + The day drill-down groups runs by the UTC calendar day they started. It ignores a day outside the active trend window. The day CSV repeats the active agent in every row. The all-agents view leaves that cell empty. @@ -1382,13 +1463,14 @@ The span exporter sends each workbench span as it ends. It does not batch spans. - Release 1.5 complete: add a CSV export for the failure trend and an agent-level trend filter on the dashboard. - Release 1.6 complete: add a trend window selector and a per-day drill-down on the dashboard chart. - Release 1.7 complete: add a status breakdown beside the daily failure line on the dashboard. -- Release 1.8: add an agent comparison overlay to the failure trend. +- Release 1.8 complete: add an agent comparison overlay to the failure trend. +- Release 1.9: add a run-level error timeline to the run detail page. ## Repository map `fixtures/` contains meaningful baseline, candidate, and second-agent traces. It also contains a handler config and demo scripts. -`tests/` contains deterministic tests for the core. It covers coordination, guards, search, annotations, OTLP, export, review, reports, retention cleanup, scheduled cleanup, the server scheduler, and the failure trend, including the agent filter, the window selector, the day drill-down, the status breakdown, and the CSV exports. +`tests/` contains deterministic tests for the core. It covers coordination, guards, search, annotations, OTLP, export, review, reports, retention cleanup, scheduled cleanup, the server scheduler, and the failure trend, including the agent filter, the window selector, the day drill-down, the status breakdown, the agent comparison overlay, and the CSV exports. `static/` and `templates/` contain the presentation layer. diff --git a/agent_trace_workbench/__init__.py b/agent_trace_workbench/__init__.py index 16839b5..2cb3fbb 100644 --- a/agent_trace_workbench/__init__.py +++ b/agent_trace_workbench/__init__.py @@ -1,3 +1,3 @@ """Agent Trace Workbench package.""" -__version__ = "1.7.0" +__version__ = "1.8.0" diff --git a/agent_trace_workbench/cli.py b/agent_trace_workbench/cli.py index 19270f0..3c13e6b 100644 --- a/agent_trace_workbench/cli.py +++ b/agent_trace_workbench/cli.py @@ -18,6 +18,7 @@ report_to_csv, run_tools_to_csv, status_trend_to_csv, + trend_overlay_to_csv, trend_to_csv, ) from .handlers import ReplayPolicy, load_handler_config @@ -155,6 +156,11 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="Restrict the trend to one agent name", ) + trend.add_argument( + "--compare", + default=None, + help="Draw a second failure line for one agent comparison", + ) trend.add_argument( "--format", choices=["json", "csv"], @@ -424,6 +430,16 @@ def main() -> None: ) elif args.days < 1 or args.days > 90: raise SystemExit("--days must be between 1 and 90") + elif args.compare: + if args.compare == (args.agent or ""): + raise SystemExit("--compare must differ from --agent") + overlay = store.failure_trend_overlay( + args.days, agent_name=args.agent, compare_agent=args.compare + ) + if args.format == "csv": + print(trend_overlay_to_csv(overlay), end="") + else: + print(json.dumps(overlay, indent=2)) elif args.statuses: buckets = store.status_trend(args.days, agent_name=args.agent) if args.format == "csv": diff --git a/agent_trace_workbench/export.py b/agent_trace_workbench/export.py index fea702a..c7db478 100644 --- a/agent_trace_workbench/export.py +++ b/agent_trace_workbench/export.py @@ -69,6 +69,15 @@ _TREND_HEADERS = ["day", "agent_name", "runs", "failures", "failure_rate"] +_TREND_OVERLAY_HEADERS = [ + "day", + "series", + "agent_name", + "runs", + "failures", + "failure_rate", +] + _STATUS_TREND_HEADERS = ["day", "agent_name", "status", "runs"] _DAY_RUNS_HEADERS = [ @@ -266,6 +275,41 @@ def trend_to_csv(trend: list[dict[str, Any]], agent_name: str = "") -> str: return _to_csv(_TREND_HEADERS, rows) +def trend_overlay_to_csv(overlay: dict[str, Any]) -> str: + """Render an agent comparison overlay as a CSV document. + + The document lists one row per day per series. A series column marks + each row as the primary line or the compare line, so a spreadsheet + can plot both series from one file. Empty days stay in both series, + because the overlay shares one time axis. + """ + + primary = overlay["primary"] + compare = overlay["compare"] + attributes = { + "trend.days": len(primary), + "trend.compare": overlay.get("compare_agent", ""), + } + with traced_operation("export.trend_overlay_csv", attributes): + rows: list[dict[str, Any]] = [] + for series, buckets, agent_name in ( + ("primary", primary, overlay.get("primary_agent", "")), + ("compare", compare, overlay.get("compare_agent", "")), + ): + for bucket in buckets: + rows.append( + { + "day": bucket["day"], + "series": series, + "agent_name": agent_name, + "runs": bucket["runs"], + "failures": bucket["failures"], + "failure_rate": _number(bucket.get("failure_rate")), + } + ) + return _to_csv(_TREND_OVERLAY_HEADERS, rows) + + def status_trend_to_csv( trend: list[dict[str, Any]], agent_name: str = "" ) -> str: diff --git a/agent_trace_workbench/main.py b/agent_trace_workbench/main.py index 0e905fa..1527652 100644 --- a/agent_trace_workbench/main.py +++ b/agent_trace_workbench/main.py @@ -25,6 +25,7 @@ report_to_csv, run_tools_to_csv, status_trend_to_csv, + trend_overlay_to_csv, trend_to_csv, ) from .handlers import ReplayPolicy, load_handler_config @@ -84,11 +85,28 @@ def dashboard( agent: str | None = Query(default=None, max_length=200), days: int = Query(default=14, ge=1, le=90), day: str | None = Query(default=None, max_length=10), + compare: str | None = Query(default=None, max_length=200), ) -> Any: runs = app.state.store.search_runs(q) if q else app.state.store.list_runs() selected_agent = agent or "" - trend = app.state.store.failure_trend(days, agent_name=selected_agent or None) - chart = _trend_chart(trend) + compare_agent = compare or "" + overlay = None + if compare_agent and compare_agent != selected_agent: + overlay = app.state.store.failure_trend_overlay( + days, + agent_name=selected_agent or None, + compare_agent=compare_agent, + ) + trend = overlay["primary"] + else: + trend = app.state.store.failure_trend( + days, agent_name=selected_agent or None + ) + chart = _trend_chart( + trend, + compare_trend=overlay["compare"] if overlay else None, + compare_agent=compare_agent if overlay else "", + ) for point in chart["points"]: point["href"] = _day_href(selected_agent, days, point["day"]) day_names = {point["day"] for point in chart["points"]} @@ -103,6 +121,7 @@ def dashboard( status_breakdown = app.state.store.status_trend( days, agent_name=selected_agent or None ) + trend_agents = app.state.store.trend_agents() return render_template( request, "dashboard.html", @@ -112,8 +131,14 @@ def dashboard( "trend": chart, "status_bars": _status_bars(status_breakdown), "query": q or "", - "trend_agents": app.state.store.trend_agents(), + "trend_agents": trend_agents, + "compare_agents": ( + [item for item in trend_agents if item != selected_agent] + if len(trend_agents) >= 2 + else [] + ), "selected_agent": selected_agent, + "compare_agent": compare_agent if overlay else "", "selected_day": selected_day, "day_runs": day_runs, "day_csv_link": ( @@ -123,6 +148,11 @@ def dashboard( ), "trend_links": _trend_links(selected_agent, days), "status_links": _status_links(selected_agent, days), + "overlay_links": ( + _overlay_links(selected_agent, compare_agent, days) + if overlay + else None + ), "store": app.state.store.store_info(), "telemetry": _telemetry_info(), "scheduler": _scheduler_status(app), @@ -388,6 +418,34 @@ def api_trend( def api_trend_agents() -> list[str]: return app.state.store.trend_agents() + @app.get("/api/trend/overlay", response_model=None) + def api_trend_overlay( + days: int = Query(default=14, ge=1, le=90), + agent: str | None = Query(default=None, max_length=200), + compare: str | None = Query(default=None, max_length=200), + export_format: str = Query(default="json", alias="format"), + ) -> Response | dict[str, Any]: + if not compare or not compare.strip(): + raise HTTPException( + status_code=400, detail="compare must name an agent" + ) + if agent and compare == agent: + raise HTTPException( + status_code=400, detail="compare must differ from agent" + ) + overlay = app.state.store.failure_trend_overlay( + days, agent_name=agent, compare_agent=compare + ) + if export_format == "csv": + return _download_response( + trend_overlay_to_csv(overlay), + "failure-trend-overlay.csv", + "text/csv; charset=utf-8", + ) + if export_format != "json": + raise HTTPException(status_code=400, detail="format must be 'json' or 'csv'") + return overlay + @app.get("/api/trend/statuses", response_model=None) def api_trend_statuses( days: int = Query(default=14, ge=1, le=90), @@ -776,12 +834,19 @@ def _scheduler_status(app: FastAPI) -> dict[str, Any]: return scheduler.status() -def _trend_chart(trend: list[dict[str, Any]]) -> dict[str, Any]: +def _trend_chart( + trend: list[dict[str, Any]], + *, + compare_trend: list[dict[str, Any]] | None = None, + compare_agent: str = "", +) -> dict[str, Any]: """Shape a failure trend into SVG-ready data for the dashboard. The helper maps each day to a point on a fixed view box. It returns the point list, a few day labels, and the window totals so the template can draw one line chart without any charting dependency. + Pass compare_trend and compare_agent to add a second line on the same + time axis. Both series share one day window, so their points line up. """ width = 680 @@ -790,26 +855,15 @@ def _trend_chart(trend: list[dict[str, Any]]) -> dict[str, Any]: pad_y = 10 count = len(trend) step = (width - 2 * pad_x) / max(count - 1, 1) - points = [] - for index, item in enumerate(trend): - x = round(pad_x + index * step, 2) - rate = min(item["failure_rate"], 1.0) - y = round(height - pad_y - rate * (height - 2 * pad_y), 2) - points.append({**item, "x": x, "y": y}) + points = _trend_points(trend, width, height, pad_x, pad_y, step) label_indices = sorted({0, count - 1, count // 2}) if count else [] labels = [ {"x": points[index]["x"], "day": points[index]["day"]} for index in label_indices if index < count ] - totals = { - "runs": sum(item["runs"] for item in trend), - "failures": sum(item["failures"] for item in trend), - } - totals["failure_rate"] = ( - round(totals["failures"] / totals["runs"], 4) if totals["runs"] else 0.0 - ) - return { + totals = _trend_totals(trend) + chart: dict[str, Any] = { "width": width, "height": height, "days": count, @@ -818,6 +872,48 @@ def _trend_chart(trend: list[dict[str, Any]]) -> dict[str, Any]: "active_days": sum(1 for item in trend if item["runs"] > 0), "totals": totals, } + if compare_trend is not None: + compare_points = _trend_points( + compare_trend, width, height, pad_x, pad_y, step + ) + chart["compare"] = { + "agent": compare_agent, + "points": compare_points, + "totals": _trend_totals(compare_trend), + } + return chart + + +def _trend_points( + trend: list[dict[str, Any]], + width: int, + height: int, + pad_x: int, + pad_y: int, + step: float, +) -> list[dict[str, Any]]: + """Map each trend bucket to a point on the shared SVG view box.""" + + points = [] + for index, item in enumerate(trend): + x = round(pad_x + index * step, 2) + rate = min(item["failure_rate"], 1.0) + y = round(height - pad_y - rate * (height - 2 * pad_y), 2) + points.append({**item, "x": x, "y": y}) + return points + + +def _trend_totals(trend: list[dict[str, Any]]) -> dict[str, Any]: + """Return the window totals for one failure trend series.""" + + totals = { + "runs": sum(item["runs"] for item in trend), + "failures": sum(item["failures"] for item in trend), + } + totals["failure_rate"] = ( + round(totals["failures"] / totals["runs"], 4) if totals["runs"] else 0.0 + ) + return totals def _trend_links(selected_agent: str, days: int) -> dict[str, str]: @@ -864,6 +960,27 @@ def _status_links(selected_agent: str, days: int) -> dict[str, str]: } +def _overlay_links(selected_agent: str, compare_agent: str, days: int) -> dict[str, str]: + """Return the JSON and CSV export links for the agent comparison overlay. + + The links keep the primary agent, the compare agent, and any + non-default window size, so a download matches what the panel draws. + Agent names are URL-encoded because they may contain spaces or + punctuation. + """ + + params: dict[str, Any] = {"compare": compare_agent} + if selected_agent: + params["agent"] = selected_agent + if days != 14: + params["days"] = days + prefix = urlencode(params) + return { + "json": f"/api/trend/overlay?{prefix}", + "csv": f"/api/trend/overlay?{prefix}&format=csv", + } + + _STATUS_PRIORITY = {"ok": 0, "error": 1, "unset": 2} diff --git a/agent_trace_workbench/storage.py b/agent_trace_workbench/storage.py index c39d858..e0094da 100644 --- a/agent_trace_workbench/storage.py +++ b/agent_trace_workbench/storage.py @@ -772,6 +772,40 @@ def failure_trend( ) return buckets + def failure_trend_overlay( + self, + days: int = 14, + *, + agent_name: str | None = None, + compare_agent: str, + ) -> dict[str, Any]: + """Return two daily failure trends for an agent comparison overlay. + + The primary series follows the same rule as failure_trend: it + covers every recorded agent unless agent_name is set. The compare + series covers compare_agent only. Both series share one UTC + calendar window, so the dashboard can draw them on the same time + axis. The overlay rejects an empty compare agent and a compare + agent equal to the primary filter, because those would draw one + line twice. + """ + + if not compare_agent or not compare_agent.strip(): + raise ValueError("compare_agent must not be empty") + if agent_name and compare_agent == agent_name: + raise ValueError("compare_agent must differ from agent_name") + attributes = {"trend.days": days, "trend.compare": compare_agent} + with traced_operation("storage.failure_trend_overlay", attributes): + primary = self.failure_trend(days, agent_name=agent_name) + compare = self.failure_trend(days, agent_name=compare_agent) + return { + "days": len(primary), + "primary_agent": agent_name or "", + "compare_agent": compare_agent, + "primary": primary, + "compare": compare, + } + def status_trend( self, days: int = 14, diff --git a/pyproject.toml b/pyproject.toml index bac403a..81f1a2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "agent-trace-workbench" -version = "1.7.0" +version = "1.8.0" description = "A local workbench for recording, replaying, comparing, and inspecting agent traces." readme = "README.md" requires-python = ">=3.11" diff --git a/static/styles.css b/static/styles.css index e4444e9..c602f7a 100644 --- a/static/styles.css +++ b/static/styles.css @@ -206,7 +206,12 @@ input[type="checkbox"] { width: 16px; height: 16px; accent-color: var(--navy); c .trend-chart { position: relative; } .trend-svg { display: block; width: 100%; height: 150px; overflow: visible; } .trend-line { fill: none; stroke: var(--coral); stroke-width: 2; stroke-linecap: round; stroke-linejoin: round; } +.trend-line-compare { stroke: var(--blue); stroke-width: 2; stroke-dasharray: 5 4; } .trend-dot { fill: var(--coral); stroke: #fff; stroke-width: 1.2; transition: r .12s ease; } +.trend-dot-compare { fill: var(--blue); stroke: #fff; stroke-width: 1.2; pointer-events: none; } +.swatch-primary { background: var(--coral); } +.swatch-compare { background: var(--blue); } +.trend-legend-overlay { margin-top: 14px; padding-top: 14px; } .trend-dot-link { cursor: pointer; } .trend-dot-link:hover .trend-dot, .trend-dot-link:focus-visible .trend-dot { r: 5.2; fill: var(--navy); } .trend-dot-link:focus-visible { outline: 0; } diff --git a/templates/dashboard.html b/templates/dashboard.html index 7deccfc..dbd5b16 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -2,7 +2,7 @@ {% block title %}Runs | Agent Trace Workbench{% endblock %} {% block content %}
-
LOCAL OBSERVABILITY / RELEASE 1.7
+
LOCAL OBSERVABILITY / RELEASE 1.8

See what your agent did.
Replay why it did it.

Ingest local JSON traces, import OTLP exports, watch a folder, inspect every tool boundary, replay recorded work, compare runs, label evidence for review, and read a folder-level library report without a hosted service.

@@ -50,7 +50,7 @@

See what your agent did.
Replay why it did it.

-
03 / FAILURE TREND

Do failures rise or fall?

last {{ trend.days }} days · {{ trend.active_days }} with activity{% if selected_agent %} · {{ selected_agent }}{% endif %}
+
03 / FAILURE TREND

Do failures rise or fall?

last {{ trend.days }} days · {{ trend.active_days }} with activity{% if selected_agent %} · {{ selected_agent }}{% endif %}{% if trend.compare %} vs {{ trend.compare.agent }}{% endif %}
{% if trend_agents %} {% endif %} {% if trend.active_days %}
-
@@ -145,6 +165,7 @@

{{ run.agent_name }}{% if run.label %} {{ ru {% if selected_agent %}{% endif %} + {% if compare_agent %}{% endif %} {% if trend.days != 14 %}{% endif %} {% if selected_day %}{% endif %} diff --git a/tests/test_trend_overlay.py b/tests/test_trend_overlay.py new file mode 100644 index 0000000..97cf838 --- /dev/null +++ b/tests/test_trend_overlay.py @@ -0,0 +1,396 @@ +"""Deterministic tests for the agent comparison overlay on the failure trend.""" + +import json +import sqlite3 +from datetime import datetime, timedelta, timezone + +import pytest +from fastapi.testclient import TestClient + +from agent_trace_workbench.cli import main +from agent_trace_workbench.main import create_app +from agent_trace_workbench.storage import TraceStore + + +def _day(days_ago: int) -> str: + """Return a UTC calendar day relative to today for stable buckets.""" + return (datetime.now(timezone.utc) - timedelta(days=days_ago)).strftime( + "%Y-%m-%d" + ) + + +def _set_started(store: TraceStore, run_id: str, days_ago: int) -> None: + day = _day(days_ago) + with sqlite3.connect(store.db_path) as connection: + connection.execute( + "UPDATE runs SET started_at = ?, ended_at = ? WHERE run_id = ?", + (f"{day}T09:00:00+00:00", f"{day}T09:00:10+00:00", run_id), + ) + + +def _series_bucket(series, days_ago: int) -> dict: + day = _day(days_ago) + return next(item for item in series if item["day"] == day) + + +def test_overlay_returns_two_series(tmp_path, baseline, candidate, support): + store = TraceStore(tmp_path / "overlay.db") + store.ingest(baseline, "baseline.json") + store.ingest(candidate, "candidate.json") + store.ingest(support, "support.json") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 2) + _set_started(store, support.run_id, 1) + + overlay = store.failure_trend_overlay( + 7, agent_name="catalog-assistant", compare_agent="support-assistant" + ) + + assert overlay["days"] == 7 + assert overlay["primary_agent"] == "catalog-assistant" + assert overlay["compare_agent"] == "support-assistant" + catalog = _series_bucket(overlay["primary"], 2) + assert catalog["runs"] == 2 + assert catalog["failures"] == 1 + support_bucket = _series_bucket(overlay["compare"], 1) + assert support_bucket["runs"] == 1 + assert support_bucket["failures"] == 0 + assert _series_bucket(overlay["compare"], 2)["runs"] == 0 + + +def test_overlay_primary_covers_all_agents_when_unfiltered( + tmp_path, baseline, candidate, support +): + store = TraceStore(tmp_path / "overlay.db") + store.ingest(baseline, "baseline.json") + store.ingest(candidate, "candidate.json") + store.ingest(support, "support.json") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 2) + _set_started(store, support.run_id, 2) + + overlay = store.failure_trend_overlay( + 7, agent_name=None, compare_agent="support-assistant" + ) + + assert overlay["primary_agent"] == "" + assert _series_bucket(overlay["primary"], 2)["runs"] == 3 + + +def test_overlay_keeps_empty_days_in_both_series(tmp_path, baseline, support): + store = TraceStore(tmp_path / "overlay.db") + store.ingest(baseline, "baseline.json") + store.ingest(support, "support.json") + _set_started(store, baseline.run_id, 5) + _set_started(store, support.run_id, 5) + + overlay = store.failure_trend_overlay( + 7, agent_name="catalog-assistant", compare_agent="support-assistant" + ) + + assert len(overlay["primary"]) == 7 + assert len(overlay["compare"]) == 7 + assert _series_bucket(overlay["primary"], 1)["runs"] == 0 + assert _series_bucket(overlay["compare"], 1)["failure_rate"] == 0.0 + + +def test_overlay_rejects_empty_compare(tmp_path, baseline): + store = TraceStore(tmp_path / "overlay.db") + store.ingest(baseline, "baseline.json") + + with pytest.raises(ValueError, match="compare_agent"): + store.failure_trend_overlay(7, agent_name=None, compare_agent=" ") + + +def test_overlay_rejects_compare_equal_to_primary(tmp_path, baseline): + store = TraceStore(tmp_path / "overlay.db") + store.ingest(baseline, "baseline.json") + + with pytest.raises(ValueError, match="must differ"): + store.failure_trend_overlay( + 7, agent_name="catalog-assistant", compare_agent="catalog-assistant" + ) + + +def test_overlay_caps_large_windows(tmp_path, baseline, support): + store = TraceStore(tmp_path / "overlay.db") + store.ingest(baseline, "baseline.json") + store.ingest(support, "support.json") + + overlay = store.failure_trend_overlay( + 500, agent_name=None, compare_agent="support-assistant" + ) + + assert overlay["days"] == 90 + assert len(overlay["primary"]) == 90 + assert len(overlay["compare"]) == 90 + + +def test_api_overlay_returns_both_series(tmp_path, baseline, candidate, support): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=candidate.as_jsonable()) + client.post("/api/traces", json=support.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 2) + _set_started(store, support.run_id, 1) + + body = client.get( + "/api/trend/overlay", + params={ + "days": 7, + "agent": "catalog-assistant", + "compare": "support-assistant", + }, + ).json() + + assert body["compare_agent"] == "support-assistant" + assert body["primary_agent"] == "catalog-assistant" + assert len(body["primary"]) == 7 + assert _series_bucket(body["primary"], 2)["runs"] == 2 + assert _series_bucket(body["compare"], 1)["runs"] == 1 + + +def test_api_overlay_csv_returns_attachment(tmp_path, baseline, support): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=support.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 2) + _set_started(store, support.run_id, 1) + + response = client.get( + "/api/trend/overlay", + params={"days": 7, "compare": "support-assistant", "format": "csv"}, + ) + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/csv") + assert response.headers["content-disposition"] == ( + 'attachment; filename="failure-trend-overlay.csv"' + ) + lines = response.text.splitlines() + assert lines[0] == "day,series,agent_name,runs,failures,failure_rate" + primary_rows = [ + line for line in lines[1:] if line.startswith(f"{_day(2)},primary,") + ] + assert len(primary_rows) == 1 + assert primary_rows[0] == f"{_day(2)},primary,,1,0,0.0" + assert any( + line.startswith(f"{_day(1)},compare,support-assistant,") + for line in lines[1:] + ) + + +def test_api_overlay_rejects_missing_compare(tmp_path): + client = TestClient(create_app(tmp_path / "api.db")) + + assert ( + client.get("/api/trend/overlay", params={"days": 7}).status_code == 400 + ) + + +def test_api_overlay_rejects_compare_equal_to_agent(tmp_path): + client = TestClient(create_app(tmp_path / "api.db")) + + response = client.get( + "/api/trend/overlay", + params={"days": 7, "agent": "catalog", "compare": "catalog"}, + ) + + assert response.status_code == 400 + assert "differ" in response.json()["detail"] + + +def test_api_overlay_rejects_unknown_format(tmp_path): + client = TestClient(create_app(tmp_path / "api.db")) + + response = client.get( + "/api/trend/overlay", + params={"days": 7, "compare": "support", "format": "xml"}, + ) + + assert response.status_code == 400 + + +def test_api_overlay_does_not_shadow_day_route(tmp_path, baseline): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 2) + + day_body = client.get(f"/api/trend/{_day(2)}").json() + + assert day_body["day"] == _day(2) + assert [run["run_id"] for run in day_body["runs"]] == [baseline.run_id] + + +def test_cli_overlay_prints_both_series(tmp_path, baseline, support, monkeypatch, capsys): + store = TraceStore(tmp_path / "cli.db") + store.ingest(baseline, "baseline.json") + store.ingest(support, "support.json") + _set_started(store, baseline.run_id, 2) + _set_started(store, support.run_id, 1) + + monkeypatch.setattr( + "sys.argv", + [ + "atw", + "--db", + str(tmp_path / "cli.db"), + "trend", + "--days", + "7", + "--agent", + "catalog-assistant", + "--compare", + "support-assistant", + ], + ) + main() + + overlay = json.loads(capsys.readouterr().out) + assert overlay["compare_agent"] == "support-assistant" + assert overlay["primary_agent"] == "catalog-assistant" + assert _series_bucket(overlay["primary"], 2)["runs"] == 1 + assert _series_bucket(overlay["compare"], 1)["runs"] == 1 + + +def test_cli_overlay_prints_csv(tmp_path, baseline, support, monkeypatch, capsys): + store = TraceStore(tmp_path / "cli.db") + store.ingest(baseline, "baseline.json") + store.ingest(support, "support.json") + _set_started(store, baseline.run_id, 2) + _set_started(store, support.run_id, 1) + + monkeypatch.setattr( + "sys.argv", + [ + "atw", + "--db", + str(tmp_path / "cli.db"), + "trend", + "--days", + "7", + "--compare", + "support-assistant", + "--format", + "csv", + ], + ) + main() + + lines = capsys.readouterr().out.splitlines() + assert lines[0] == "day,series,agent_name,runs,failures,failure_rate" + assert lines[1].endswith(",primary,,0,0,0.0") + + +def test_cli_overlay_rejects_compare_equal_to_agent( + tmp_path, baseline, monkeypatch +): + store = TraceStore(tmp_path / "cli.db") + store.ingest(baseline, "baseline.json") + + monkeypatch.setattr( + "sys.argv", + [ + "atw", + "--db", + str(tmp_path / "cli.db"), + "trend", + "--agent", + "catalog-assistant", + "--compare", + "catalog-assistant", + ], + ) + with pytest.raises(SystemExit, match="must differ"): + main() + + +def test_dashboard_shows_compare_select_for_two_agents(tmp_path, baseline, support): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=support.as_jsonable()) + + page = client.get("/").text + + assert 'id="trend-compare"' in page + assert 'name="compare"' in page + assert "Compare against" in page + + +def test_dashboard_omits_compare_select_for_one_agent(tmp_path, baseline): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + + page = client.get("/").text + + assert 'id="trend-compare"' not in page + + +def test_dashboard_draws_comparison_overlay(tmp_path, baseline, candidate, support): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=candidate.as_jsonable()) + client.post("/api/traces", json=support.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 2) + _set_started(store, support.run_id, 1) + + page = client.get( + "/", + params={"agent": "catalog-assistant", "compare": "support-assistant"}, + ).text + + assert "trend-line-compare" in page + assert "trend-dot-compare" in page + assert "swatch-compare" in page + assert "vs support-assistant" in page + assert "failure rate per series" in page + assert ( + "/api/trend/overlay?compare=support-assistant&agent=catalog-assistant&format=csv" + in page + ) + assert 'value="support-assistant" selected' in page + + +def test_dashboard_overlay_all_agents_view(tmp_path, baseline, candidate, support): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=candidate.as_jsonable()) + client.post("/api/traces", json=support.as_jsonable()) + + page = client.get("/", params={"compare": "support-assistant"}).text + + assert "All agents" in page + assert "trend-line-compare" in page + assert ( + "/api/trend/overlay?compare=support-assistant&format=csv" in page + ) + + +def test_dashboard_ignores_compare_equal_to_selected_agent( + tmp_path, baseline, support +): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=support.as_jsonable()) + + page = client.get( + "/", params={"agent": "support-assistant", "compare": "support-assistant"} + ).text + + assert "trend-line-compare" not in page + assert "swatch-compare" not in page + + +def test_dashboard_empty_store_omits_overlay(tmp_path): + client = TestClient(create_app(tmp_path / "api.db")) + + page = client.get("/", params={"compare": "support-assistant"}).text + + assert "trend-svg" not in page + assert 'id="trend-compare"' not in page