From 9f026b1d31659604bba2000c4e3149ee82d3d175 Mon Sep 17 00:00:00 2001 From: DanieCuevas <43822444+DanielCuevas1208@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:54:18 -0700 Subject: [PATCH] feat: extend agent trace workbench --- CHANGELOG.md | 23 ++++ README.md | 101 +++++++++++++- agent_trace_workbench/__init__.py | 2 +- agent_trace_workbench/cli.py | 27 +++- agent_trace_workbench/export.py | 39 ++++++ agent_trace_workbench/main.py | 110 +++++++++++++-- agent_trace_workbench/storage.py | 34 ++++- pyproject.toml | 2 +- static/styles.css | 8 +- templates/dashboard.html | 38 +++++- tests/test_csv_export.py | 86 ++++++++++++ tests/test_trend.py | 220 ++++++++++++++++++++++++++++++ 12 files changed, 662 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06e230a..2c1404a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,29 @@ All notable changes to Agent Trace Workbench appear in this file. The version format follows a release cycle. A release adds one coherent capability to the workbench. +## 1.6.0 - 2026-08-04 + +### Added + +- A trend window selector on the dashboard chart for 7, 14, 30, and 90 day views. +- A per-day trend drill-down that lists the runs started on one UTC calendar day. +- Clickable day dots on the trend chart that open the drill-down for that day. +- A day panel on the dashboard that shows the day runs and links each one to its page. +- `GET /api/trend?days=` option that sets the trend window for scripts. +- `GET /api/trend/{day}` route that returns the runs started on one day, with an `agent` filter. +- `GET /api/trend/{day}?format=csv` route that returns the day runs as a CSV attachment. +- `atw trend --day ` command that lists the runs started on one day. +- `atw trend --day --format csv` command that prints the day runs as CSV. +- The dashboard export links keep the active window size and agent filter. +- Deterministic tests for the window selector, the drill-down, the day routes, the day CSV, and the CLI options. + +### Changed + +- Version numbers moved to 1.6.0. +- The dashboard trend filter now carries a window selector beside the agent filter. +- The dashboard keeps the selected day and window when it searches or changes filters. +- The architecture now includes a per-day trend drill-down beside the failure trend. + ## 1.5.0 - 2026-08-04 ### Added diff --git a/README.md b/README.md index ebe8108..8b3a015 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ Release 1.4 adds a server-side sweep scheduler and a failure trend line on the d Release 1.5 adds a CSV export for the failure trend and an agent-level trend filter on the dashboard. Filter the chart to one agent. Download the same series as CSV from the panel, the API, or the CLI. +Release 1.6 adds a trend window selector and a per-day drill-down on the dashboard. Choose 7, 14, 30, or 90 day views. Click a day to see the runs that started that day. + ## Value Agent debugging needs evidence at tool boundaries. @@ -62,12 +64,12 @@ SQLite runs in WAL mode with a busy timeout. Readers keep a committed snapshot. - `models.py` defines the portable trace contract. - `handlers.py` loads local handler config and applies side-effect guards. -- `storage.py` owns the SQLite schema, WAL coordination, idempotent ingestion, and local annotations. It also computes the review list, applies bulk labels, builds the library report, and enforces the retention cutoff for cleanup. A cleanup log records each scheduled sweep. +- `storage.py` owns the SQLite schema, WAL coordination, idempotent ingestion, and local annotations. It also computes the review list, applies bulk labels, builds the library report, computes the daily failure trend, lists the runs for one day, and enforces the retention cutoff for cleanup. A cleanup log records each scheduled sweep. - `ingestion.py` watches JSON files and returns stable schema error reports. - `otlp.py` converts the OTLP JSON encoding to and from the trace contract. - `replay.py` runs guarded local handlers and records mismatches. - `compare.py` aligns tool calls by recorded position and reports field-level deltas. -- `export.py` renders comparisons, run tool calls, library reports, and failure trends as CSV files. +- `export.py` renders comparisons, run tool calls, library reports, failure trends, and day run lists as CSV files. - `collector.py` posts recorded runs to a local collector over OTLP HTTP JSON. - `main.py` serves the interface and the JSON API. - `scheduler.py` runs server-side retention sweeps on an interval. @@ -324,6 +326,88 @@ python -m agent_trace_workbench.cli trend --agent catalog-assistant --format csv The dashboard panel links to both downloads. The links keep the active agent. +## Trend window + +Choose the trend window on the dashboard chart. + +Select 7, 14, 30, or 90 days beside the agent filter. The chart redraws with that window. Window totals match the selected span. + +```powershell +curl.exe "http://127.0.0.1:8000/api/trend?days=30" +``` + +The response keeps one bucket per day across the full window. Empty days stay in the series. + +The JSON and CSV export links keep the active window. The default is 14 days. + +```powershell +curl.exe "http://127.0.0.1:8000/api/trend?days=30&format=csv" +python -m agent_trace_workbench.cli trend --days 30 +``` + +## Day drill-down + +Click a day on the trend chart. The dashboard opens a panel with the runs that started that day. + +Each day dot links to that panel. The panel lists one run per card. Each card links to its run page. + +```powershell +curl.exe "http://127.0.0.1:8000/api/trend/2026-07-31" +``` + +The response lists the runs for that day. + +```json +{ + "day": "2026-07-31", + "agent": "", + "runs": [ + { + "run_id": "run-baseline-001", + "agent_name": "catalog-assistant", + "status": "ok", + "tool_count": 2 + }, + { + "run_id": "run-candidate-001", + "agent_name": "catalog-assistant", + "status": "error", + "tool_count": 3 + } + ] +} +``` + +Filter the day view to one agent. + +```powershell +curl.exe "http://127.0.0.1:8000/api/trend/2026-07-31?agent=catalog-assistant" +``` + +The day panel on the dashboard keeps the active agent. Change the agent or the window on the trend form. The day view follows both. + +Download the day runs as CSV. + +```powershell +curl.exe -o runs-2026-07-31.csv "http://127.0.0.1:8000/api/trend/2026-07-31?format=csv" +``` + +The file lists one row per run. The day cell repeats the drill target. + +```text +day,run_id,agent_name,status,tool_count,duration_ms,source_dir,label +2026-07-31,run-baseline-001,catalog-assistant,ok,2,220.0,fixtures, +``` + +Use the CLI for scripts. + +```powershell +python -m agent_trace_workbench.cli trend --day 2026-07-31 +python -m agent_trace_workbench.cli trend --day 2026-07-31 --format csv +``` + +The day panel offers the same CSV download. A day outside the active window is ignored. The dashboard draws no panel for it. + ## Saved comparisons Save a comparison for later review. @@ -1119,7 +1203,7 @@ curl.exe -X POST http://127.0.0.1:8000/api/traces ` ## Test status -The test suite covers the core flows. It covers storage, ingestion, replay, comparison, search, annotations, bulk labels, export, review, reports, retention cleanup, and scheduled cleanup. It covers the CLI, the API, collector export, the server scheduler, and the dashboard failure trend, including the agent filter and the CSV export. +The test suite covers the core flows. It covers storage, ingestion, replay, comparison, search, annotations, bulk labels, export, review, reports, retention cleanup, and scheduled cleanup. It covers the CLI, the API, collector export, the server scheduler, and the dashboard failure trend, including the agent filter, the window selector, the day drill-down, and the CSV exports. Run the checks with these commands. @@ -1130,7 +1214,7 @@ python scripts/check_requirements.py python -m compileall agent_trace_workbench tests ``` -Current verification passes 268 tests, Ruff lint, dependency checks, and Python compilation. CI installs from `requirements-lock.txt` and runs these checks on Python 3.11, 3.12, and 3.13 for every push and pull request. +Current verification passes 289 tests, Ruff lint, dependency checks, and Python compilation. CI installs from `requirements-lock.txt` and runs these checks on Python 3.11, 3.12, and 3.13 for every push and pull request. ## Limitations @@ -1172,6 +1256,10 @@ The trend CSV repeats the active agent in every row. The all-agents view leaves The trend export lists one row per day. It does not add a window total row. +The day drill-down groups runs by the UTC calendar day they started. It ignores a day outside the active trend window. + +The day CSV repeats the active agent in every row. The all-agents view leaves that cell empty. + The cleanup history records policy and counts. It does not store the deleted traces. The report retention line counts runs under the current policy. It uses `older_than_days` from the request or the 30-day default. @@ -1224,13 +1312,14 @@ The span exporter sends each workbench span as it ends. It does not batch spans. - Release 1.3 complete: add a scheduled cleanup run and a retention line to the library report. - Release 1.4 complete: add a server-side sweep scheduler and a failure trend line on the dashboard. - Release 1.5 complete: add a CSV export for the failure trend and an agent-level trend filter on the dashboard. -- Release 1.6: add a per-day trend drill-down and a window selector on the dashboard chart. +- Release 1.6 complete: add a trend window selector and a per-day drill-down on the dashboard chart. +- Release 1.7: add a status breakdown beside the daily failure line on the dashboard. ## Repository map `fixtures/` contains meaningful baseline, candidate, and second-agent traces. It also contains a handler config and demo scripts. -`tests/` contains deterministic tests for the core. It covers coordination, guards, search, annotations, OTLP, export, review, reports, retention cleanup, scheduled cleanup, the server scheduler, and the failure trend, including the agent filter and the CSV export. +`tests/` contains deterministic tests for the core. It covers coordination, guards, search, annotations, OTLP, export, review, reports, retention cleanup, scheduled cleanup, the server scheduler, and the failure trend, including the agent filter, the window selector, the day drill-down, and the CSV exports. `static/` and `templates/` contain the presentation layer. diff --git a/agent_trace_workbench/__init__.py b/agent_trace_workbench/__init__.py index 809f62a..88efe31 100644 --- a/agent_trace_workbench/__init__.py +++ b/agent_trace_workbench/__init__.py @@ -1,3 +1,3 @@ """Agent Trace Workbench package.""" -__version__ = "1.5.0" +__version__ = "1.6.0" diff --git a/agent_trace_workbench/cli.py b/agent_trace_workbench/cli.py index c46c180..7553226 100644 --- a/agent_trace_workbench/cli.py +++ b/agent_trace_workbench/cli.py @@ -12,7 +12,13 @@ from .collector import export_run_to_collector from .compare import compare_runs -from .export import comparison_to_csv, report_to_csv, run_tools_to_csv, trend_to_csv +from .export import ( + comparison_to_csv, + day_runs_to_csv, + report_to_csv, + run_tools_to_csv, + trend_to_csv, +) from .handlers import ReplayPolicy, load_handler_config from .ingestion import DirectoryWatcher, watch_directory from .models import TraceDocument @@ -159,6 +165,11 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="List the agent names available for filtering", ) + trend.add_argument( + "--day", + default=None, + help="List the runs that started on one YYYY-MM-DD day", + ) annotate = subparsers.add_parser( "annotate", help="Label a run and add local review notes" @@ -391,6 +402,20 @@ def main() -> None: elif args.command == "trend": if args.agents: print(json.dumps(store.trend_agents(), indent=2)) + elif args.day: + try: + runs = store.runs_on_day(args.day, agent_name=args.agent) + except ValueError: + raise SystemExit("--day must use the YYYY-MM-DD format") from None + if args.format == "csv": + print(day_runs_to_csv(args.day, runs, agent_name=args.agent or ""), end="") + else: + print( + json.dumps( + {"day": args.day, "agent": args.agent or "", "runs": runs}, + indent=2, + ) + ) elif args.days < 1 or args.days > 90: raise SystemExit("--days must be between 1 and 90") else: diff --git a/agent_trace_workbench/export.py b/agent_trace_workbench/export.py index 11b1597..cae93b9 100644 --- a/agent_trace_workbench/export.py +++ b/agent_trace_workbench/export.py @@ -69,6 +69,17 @@ _TREND_HEADERS = ["day", "agent_name", "runs", "failures", "failure_rate"] +_DAY_RUNS_HEADERS = [ + "day", + "run_id", + "agent_name", + "status", + "tool_count", + "duration_ms", + "source_dir", + "label", +] + _SECTION_TOTAL = "total" _SECTION_SOURCE = "source" _SECTION_AGENT = "agent" @@ -253,6 +264,34 @@ def trend_to_csv(trend: list[dict[str, Any]], agent_name: str = "") -> str: return _to_csv(_TREND_HEADERS, rows) +def day_runs_to_csv(day: str, runs: list[dict[str, Any]], agent_name: str = "") -> str: + """Render the runs that started on one day as a CSV document. + + The document lists one row per run. The day cell repeats the drill + target, so the file stays self-describing. The agent_name cell + repeats the active trend filter when one is set. + """ + + with traced_operation( + "export.day_csv", {"trend.day": day, "trend.agent": agent_name} + ): + rows: list[dict[str, Any]] = [] + for run in runs: + rows.append( + { + "day": day, + "run_id": run.get("run_id", ""), + "agent_name": run.get("agent_name", ""), + "status": run.get("status", ""), + "tool_count": _number(run.get("tool_count")), + "duration_ms": _number(run.get("duration_ms")), + "source_dir": run.get("source_dir", ""), + "label": run.get("label", ""), + } + ) + return _to_csv(_DAY_RUNS_HEADERS, rows) + + def _to_csv(headers: list[str], rows: list[dict[str, Any]]) -> str: buffer = io.StringIO() writer = csv.DictWriter(buffer, fieldnames=headers, lineterminator="\n") diff --git a/agent_trace_workbench/main.py b/agent_trace_workbench/main.py index d4d68ea..7619058 100644 --- a/agent_trace_workbench/main.py +++ b/agent_trace_workbench/main.py @@ -9,7 +9,7 @@ from html import escape from pathlib import Path from typing import Any -from urllib.parse import quote +from urllib.parse import urlencode from fastapi import FastAPI, HTTPException, Query, Request from fastapi.responses import HTMLResponse, Response @@ -19,7 +19,13 @@ from . import __version__ from .collector import export_run_to_collector from .compare import compare_runs -from .export import comparison_to_csv, report_to_csv, run_tools_to_csv, trend_to_csv +from .export import ( + comparison_to_csv, + day_runs_to_csv, + report_to_csv, + run_tools_to_csv, + trend_to_csv, +) from .handlers import ReplayPolicy, load_handler_config from .models import ( BulkLabelRequest, @@ -75,22 +81,42 @@ def dashboard( request: Request, q: str | None = Query(default=None, max_length=200), agent: str | None = Query(default=None, max_length=200), + days: int = Query(default=14, ge=1, le=90), + day: str | None = Query(default=None, max_length=10), ) -> Any: runs = app.state.store.search_runs(q) if q else app.state.store.list_runs() selected_agent = agent or "" + trend = app.state.store.failure_trend(days, agent_name=selected_agent or None) + chart = _trend_chart(trend) + for point in chart["points"]: + point["href"] = _day_href(selected_agent, days, point["day"]) + day_names = {point["day"] for point in chart["points"]} + selected_day = day if day in day_names else None + day_runs = ( + app.state.store.runs_on_day( + selected_day, agent_name=selected_agent or None + ) + if selected_day + else None + ) return render_template( request, "dashboard.html", { "runs": runs, "stats": _stats(runs), - "trend": _trend_chart( - app.state.store.failure_trend(agent_name=selected_agent or None) - ), + "trend": chart, "query": q or "", "trend_agents": app.state.store.trend_agents(), "selected_agent": selected_agent, - "trend_links": _trend_links(selected_agent), + "selected_day": selected_day, + "day_runs": day_runs, + "day_csv_link": ( + _day_csv_href(selected_agent, selected_day) + if selected_day + else None + ), + "trend_links": _trend_links(selected_agent, days), "store": app.state.store.store_info(), "telemetry": _telemetry_info(), "scheduler": _scheduler_status(app), @@ -356,6 +382,28 @@ def api_trend( def api_trend_agents() -> list[str]: return app.state.store.trend_agents() + @app.get("/api/trend/{day}", response_model=None) + def api_trend_day( + day: str, + agent: str | None = Query(default=None, max_length=200), + export_format: str = Query(default="json", alias="format"), + ) -> Response | dict[str, Any]: + try: + runs = app.state.store.runs_on_day(day, agent_name=agent) + except ValueError: + raise HTTPException( + status_code=400, detail="day must use the YYYY-MM-DD format" + ) from None + if export_format == "csv": + return _download_response( + day_runs_to_csv(day, runs, agent_name=agent or ""), + f"runs-{day}.csv", + "text/csv; charset=utf-8", + ) + if export_format != "json": + raise HTTPException(status_code=400, detail="format must be 'json' or 'csv'") + return {"day": day, "agent": agent or "", "runs": runs} + @app.get("/api/report", response_model=None) def api_report( export_format: str = Query(default="json", alias="format"), @@ -749,22 +797,56 @@ def _trend_chart(trend: list[dict[str, Any]]) -> dict[str, Any]: } -def _trend_links(selected_agent: str) -> dict[str, str]: +def _trend_links(selected_agent: str, days: int) -> dict[str, str]: """Return the JSON and CSV export links for the dashboard trend panel. - The links keep the active agent filter, so a download matches what - the panel draws. Agent names are URL-encoded because they may contain - spaces or punctuation. + The links keep the active agent filter and any non-default window + size, so a download matches what the panel draws. Agent names are + URL-encoded because they may contain spaces or punctuation. """ - prefix = f"?agent={quote(selected_agent)}" if selected_agent else "" - join = "&" if prefix else "?" + params: dict[str, Any] = {} + if days != 14: + params["days"] = days + if selected_agent: + params["agent"] = selected_agent + if not params: + return {"json": "/api/trend", "csv": "/api/trend?format=csv"} + prefix = urlencode(params) return { - "json": f"/api/trend{prefix}", - "csv": f"/api/trend{prefix}{join}format=csv", + "json": f"/api/trend?{prefix}", + "csv": f"/api/trend?{prefix}&format=csv", } +def _day_href(selected_agent: str, days: int, day: str) -> str: + """Return the drill-down link for one day on the trend chart. + + The link keeps the window size and the agent filter, so the day view + matches the panel that drew it. It anchors on the day panel below the + chart. + """ + + params: dict[str, Any] = {"days": days, "day": day} + if selected_agent: + params["agent"] = selected_agent + return f"/?{urlencode(params)}#trend-day" + + +def _day_csv_href(selected_agent: str, day: str) -> str: + """Return the CSV download link for one day drill-down panel. + + The link keeps the active agent filter, so the file matches what the + panel lists. Agent names are URL-encoded for the same reason as the + trend export links. + """ + + params: dict[str, Any] = {"format": "csv"} + if selected_agent: + params["agent"] = selected_agent + return f"/api/trend/{day}?{urlencode(params)}" + + def _span_filter_set( run: dict[str, Any], kind: str | None, diff --git a/agent_trace_workbench/storage.py b/agent_trace_workbench/storage.py index 9a44802..66bbcb5 100644 --- a/agent_trace_workbench/storage.py +++ b/agent_trace_workbench/storage.py @@ -21,7 +21,7 @@ import json import sqlite3 import time -from datetime import datetime, timedelta, timezone +from datetime import date, datetime, timedelta, timezone from pathlib import Path from typing import Any, Callable, TypeVar from uuid import uuid4 @@ -772,6 +772,38 @@ def failure_trend( ) return buckets + def runs_on_day( + self, + day: str, + *, + agent_name: str | None = None, + limit: int = 100, + ) -> list[dict[str, Any]]: + """Return run summaries that started on one UTC calendar day. + + The drill-down view uses this method. A reviewer clicks a day on + the trend chart and sees the runs that started that day. Pass + agent_name to keep the day view in sync with the trend filter. + The day must use the YYYY-MM-DD format. + """ + + try: + date.fromisoformat(day) + except ValueError: + raise ValueError("day must be a YYYY-MM-DD date") from None + safe_limit = max(1, min(limit, 100)) + query = "SELECT * FROM runs WHERE substr(started_at, 1, 10) = ?" + params: list[Any] = [day] + if agent_name: + query += " AND agent_name = ?" + params.append(agent_name) + query += " ORDER BY started_at DESC, run_id DESC LIMIT ?" + params.append(safe_limit) + with traced_operation("storage.runs_on_day", {"trend.day": day}): + with self._connect() as connection: + rows = connection.execute(query, params).fetchall() + return _summarize_runs(connection, rows) + def get_run( self, run_id: str, diff --git a/pyproject.toml b/pyproject.toml index 0f0c4ab..18a4412 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "agent-trace-workbench" -version = "1.5.0" +version = "1.6.0" description = "A local workbench for recording, replaying, comparing, and inspecting agent traces." readme = "README.md" requires-python = ">=3.11" diff --git a/static/styles.css b/static/styles.css index 982ac91..f43d5c2 100644 --- a/static/styles.css +++ b/static/styles.css @@ -206,7 +206,11 @@ input[type="checkbox"] { width: 16px; height: 16px; accent-color: var(--navy); c .trend-chart { position: relative; } .trend-svg { display: block; width: 100%; height: 150px; overflow: visible; } .trend-line { fill: none; stroke: var(--coral); stroke-width: 2; stroke-linecap: round; stroke-linejoin: round; } -.trend-dot { fill: var(--coral); stroke: #fff; stroke-width: 1.2; } +.trend-dot { fill: var(--coral); stroke: #fff; stroke-width: 1.2; transition: r .12s ease; } +.trend-dot-link { cursor: pointer; } +.trend-dot-link:hover .trend-dot, .trend-dot-link:focus-visible .trend-dot { r: 5.2; fill: var(--navy); } +.trend-dot-link:focus-visible { outline: 0; } +.trend-dot-link:focus-visible .trend-dot { outline: 2px solid var(--blue); outline-offset: 2px; } .trend-labels { display: flex; justify-content: space-between; margin-top: 6px; color: var(--muted); font-size: 10px; } .trend-labels span { position: relative; transform: translateX(-50%); } .trend-labels span:first-child { transform: none; } @@ -215,6 +219,8 @@ input[type="checkbox"] { width: 16px; height: 16px; accent-color: var(--navy); c .trend-foot strong { color: var(--navy); font: 500 20px Georgia, serif; letter-spacing: -.02em; } .trend-foot .trend-failures strong { color: var(--coral); } .trend-foot .button { margin-left: auto; } +.day-actions { display: flex; align-items: center; gap: 10px; margin-top: 16px; } +.day-actions .button { min-height: 42px; } .scheduler-section .section-heading { align-items: center; } .scheduler-section .badge { vertical-align: middle; } .scheduler-copy { max-width: 560px; margin-bottom: 20px; color: var(--muted); font-size: 13px; line-height: 1.55; } diff --git a/templates/dashboard.html b/templates/dashboard.html index 0872f16..9dc509b 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -2,7 +2,7 @@ {% block title %}Runs | Agent Trace Workbench{% endblock %} {% block content %}
-
LOCAL OBSERVABILITY / RELEASE 1.5
+
LOCAL OBSERVABILITY / RELEASE 1.6

See what your agent did.
Replay why it did it.

Ingest local JSON traces, import OTLP exports, watch a folder, inspect every tool boundary, replay recorded work, compare runs, label evidence for review, and read a folder-level library report without a hosted service.

@@ -58,9 +58,14 @@

See what your agent did.
Replay why it did it.

{% for agent in trend_agents %}{% endfor %} + + {% if query %}{% endif %} + {% if selected_day %}{% endif %} - {% if selected_agent %}Clear{% endif %} + {% if selected_agent or selected_day or trend.days != 14 %}Clear{% endif %} {% endif %} {% if trend.active_days %} @@ -68,7 +73,7 @@

See what your agent did.
Replay why it did it.

+{% if selected_day %} +
+
03 / DAY DRILL-DOWN

Runs on {{ selected_day }}

{% if selected_agent %}{{ selected_agent }} ยท {% endif %}{{ day_runs|length }} shown
+ {% if day_runs %} +
+ {% for run in day_runs %} + +
{{ run.status }}{{ run.duration_ms|round(1) }} ms
+

{{ run.agent_name }}{% if run.label %} {{ run.label }}{% endif %}

+

{{ run.run_id }}

+
{{ run.tool_count }} tool calls{{ run.source_dir }}
+ + {% endfor %} +
+
+ CSV + Back to trend +
+ {% else %} +
+

{% if selected_agent %}No runs for {{ selected_agent }} on {{ selected_day }}{% else %}No runs on {{ selected_day }}{% endif %}

Pick another day on the chart to drill into its evidence.

+ {% endif %} +
+{% endif %} +
04 / TRACE LIBRARY

Recent runs

{{ runs|length }} shown
diff --git a/tests/test_csv_export.py b/tests/test_csv_export.py index da6fe77..f2100b5 100644 --- a/tests/test_csv_export.py +++ b/tests/test_csv_export.py @@ -11,6 +11,7 @@ from agent_trace_workbench.compare import compare_runs from agent_trace_workbench.export import ( comparison_to_csv, + day_runs_to_csv, report_to_csv, run_tools_to_csv, trend_to_csv, @@ -405,3 +406,88 @@ def test_cli_trend_rejects_bad_days(tmp_path, baseline, monkeypatch): ) with pytest.raises(SystemExit, match="--days"): main() + + +def _day(days_ago: int) -> str: + return (datetime.now(timezone.utc) - timedelta(days=days_ago)).strftime("%Y-%m-%d") + + +def test_day_runs_csv_renders_one_row_per_run(tmp_path, baseline, candidate): + store = TraceStore(tmp_path / "day.db") + store.ingest(baseline, "baseline.json") + store.ingest(candidate, "candidate.json") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 2) + + rows = _read_csv(day_runs_to_csv(_day(2), store.runs_on_day(_day(2)))) + + assert len(rows) == 2 + assert all(row["day"] == _day(2) for row in rows) + assert {row["run_id"] for row in rows} == {baseline.run_id, candidate.run_id} + assert {row["status"] for row in rows} == {"ok", "error"} + assert {row["tool_count"] for row in rows} == {"2", "3"} + + +def test_day_runs_csv_carries_agent_filter(tmp_path, baseline, support): + store = TraceStore(tmp_path / "day.db") + store.ingest(baseline, "baseline.json") + store.ingest(support, "support.json") + _set_started(store, baseline.run_id, 2) + _set_started(store, support.run_id, 2) + + rows = _read_csv( + day_runs_to_csv( + _day(2), + store.runs_on_day(_day(2), agent_name="support-assistant"), + agent_name="support-assistant", + ) + ) + + assert [row["run_id"] for row in rows] == [support.run_id] + assert all(row["agent_name"] == "support-assistant" for row in rows) + + +def test_api_trend_day_csv_carries_agent_filter(tmp_path, baseline, support): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=support.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 2) + _set_started(store, support.run_id, 2) + + response = client.get( + f"/api/trend/{_day(2)}", + params={"agent": "support-assistant", "format": "csv"}, + ) + + assert response.status_code == 200 + rows = _read_csv(response.text) + assert [row["run_id"] for row in rows] == [support.run_id] + assert all(row["agent_name"] == "support-assistant" for row in rows) + + +def test_cli_trend_day_csv_prints_rows(tmp_path, baseline, candidate, monkeypatch, capsys): + store = TraceStore(tmp_path / "cli.db") + store.ingest(baseline, "baseline.json") + store.ingest(candidate, "candidate.json") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 2) + + monkeypatch.setattr( + "sys.argv", + [ + "atw", + "--db", + str(tmp_path / "cli.db"), + "trend", + "--day", + _day(2), + "--format", + "csv", + ], + ) + main() + + rows = _read_csv(capsys.readouterr().out) + assert len(rows) == 2 + assert rows[0]["day"] == _day(2) diff --git a/tests/test_trend.py b/tests/test_trend.py index ff25a4d..d24eeb0 100644 --- a/tests/test_trend.py +++ b/tests/test_trend.py @@ -1,11 +1,13 @@ """Deterministic tests for the daily failure trend on the dashboard.""" +import json import sqlite3 from datetime import datetime, timedelta, timezone import pytest from fastapi.testclient import TestClient +from agent_trace_workbench.cli import main from agent_trace_workbench.main import create_app from agent_trace_workbench.storage import TraceStore @@ -284,3 +286,221 @@ def test_dashboard_trend_empty_state(tmp_path): assert "No runs in this window" in page assert "trend-svg" not in page + + +def test_runs_on_day_lists_runs_for_one_day(tmp_path, baseline, candidate): + store = TraceStore(tmp_path / "trend.db") + store.ingest(baseline, "baseline.json") + store.ingest(candidate, "candidate.json") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 1) + + day_runs = store.runs_on_day(_day(2)) + + assert [run["run_id"] for run in day_runs] == [baseline.run_id] + assert day_runs[0]["tool_count"] == 2 + assert day_runs[0]["status"] == "ok" + + +def test_runs_on_day_sorts_newest_first(tmp_path, baseline, candidate): + store = TraceStore(tmp_path / "trend.db") + store.ingest(baseline, "baseline.json") + store.ingest(candidate, "candidate.json") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 2) + + day_runs = store.runs_on_day(_day(2)) + + assert len(day_runs) == 2 + + +def test_runs_on_day_filters_by_agent(tmp_path, baseline, candidate, support): + store = TraceStore(tmp_path / "trend.db") + store.ingest(baseline, "baseline.json") + store.ingest(candidate, "candidate.json") + store.ingest(support, "support.json") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 2) + _set_started(store, support.run_id, 2) + + day_runs = store.runs_on_day(_day(2), agent_name="catalog-assistant") + + assert {run["run_id"] for run in day_runs} == {baseline.run_id, candidate.run_id} + + +def test_runs_on_day_returns_empty_for_missing_day(tmp_path, baseline): + store = TraceStore(tmp_path / "trend.db") + store.ingest(baseline, "baseline.json") + _set_started(store, baseline.run_id, 2) + + assert store.runs_on_day(_day(6)) == [] + + +def test_runs_on_day_rejects_bad_format(tmp_path): + store = TraceStore(tmp_path / "trend.db") + + with pytest.raises(ValueError, match="YYYY-MM-DD"): + store.runs_on_day("07/31/2026") + + +def test_api_trend_day_returns_runs(tmp_path, baseline, candidate): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=candidate.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 1) + + body = client.get(f"/api/trend/{_day(2)}").json() + + assert body["day"] == _day(2) + assert body["agent"] == "" + assert [run["run_id"] for run in body["runs"]] == [baseline.run_id] + + +def test_api_trend_day_filters_by_agent(tmp_path, baseline, candidate, support): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=candidate.as_jsonable()) + client.post("/api/traces", json=support.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 2) + _set_started(store, support.run_id, 2) + + body = client.get( + f"/api/trend/{_day(2)}", params={"agent": "support-assistant"} + ).json() + + assert [run["run_id"] for run in body["runs"]] == [support.run_id] + + +def test_api_trend_day_rejects_bad_format(tmp_path): + client = TestClient(create_app(tmp_path / "api.db")) + + assert client.get("/api/trend/not-a-day").status_code == 400 + + +def test_api_trend_day_csv_returns_attachment(tmp_path, baseline, candidate): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=candidate.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 2) + + response = client.get(f"/api/trend/{_day(2)}", params={"format": "csv"}) + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/csv") + assert response.headers["content-disposition"] == ( + f'attachment; filename="runs-{_day(2)}.csv"' + ) + assert ( + response.text.splitlines()[0] + == "day,run_id,agent_name,status,tool_count,duration_ms,source_dir,label" + ) + + +def test_dashboard_shows_window_selector(tmp_path, baseline, candidate): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=candidate.as_jsonable()) + + page = client.get("/").text + + assert "trend-days" in page + assert 'value="14" selected' in page + assert 'value="90"' in page + + +def test_dashboard_trend_respects_days_window(tmp_path, baseline, candidate): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=candidate.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 2) + + page = client.get("/", params={"days": 30}).text + + assert 'value="30" selected' in page + assert "last 30 days" in page + assert "30 days" in page + + +def test_dashboard_day_drill_down_lists_runs(tmp_path, baseline, candidate): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=candidate.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 2) + + page = client.get("/", params={"day": _day(2)}).text + + assert "DAY DRILL-DOWN" in page + assert f"Runs on {_day(2)}" in page + assert baseline.run_id in page + assert f"/api/trend/{_day(2)}?" in page + + +def test_dashboard_day_drill_down_empty_state(tmp_path, baseline): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 2) + + page = client.get("/", params={"day": _day(6)}).text + + assert "DAY DRILL-DOWN" in page + assert f"No runs on {_day(6)}" in page + + +def test_dashboard_ignores_day_outside_window(tmp_path, baseline): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 40) + + page = client.get("/", params={"days": 7, "day": _day(40)}).text + + assert "DAY DRILL-DOWN" not in page + + +def test_dashboard_drill_down_links_from_chart(tmp_path, baseline): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + store = TraceStore(tmp_path / "api.db") + _set_started(store, baseline.run_id, 2) + + page = client.get("/").text + + assert f'href="/?days=14&day={_day(2)}#trend-day"' in page + + +def test_cli_trend_day_lists_runs(tmp_path, baseline, candidate, monkeypatch, capsys): + store = TraceStore(tmp_path / "cli.db") + store.ingest(baseline, "baseline.json") + store.ingest(candidate, "candidate.json") + _set_started(store, baseline.run_id, 2) + _set_started(store, candidate.run_id, 1) + + monkeypatch.setattr( + "sys.argv", + ["atw", "--db", str(tmp_path / "cli.db"), "trend", "--day", _day(2)], + ) + main() + + body = json.loads(capsys.readouterr().out) + assert body["day"] == _day(2) + assert [run["run_id"] for run in body["runs"]] == [baseline.run_id] + + +def test_cli_trend_day_rejects_bad_format(tmp_path, monkeypatch, capsys): + monkeypatch.setattr( + "sys.argv", + ["atw", "--db", str(tmp_path / "cli.db"), "trend", "--day", "07/31/2026"], + ) + with pytest.raises(SystemExit, match="YYYY-MM-DD"): + main()