diff --git a/CHANGELOG.md b/CHANGELOG.md index 3319ed7..6d74243 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to Agent Trace Workbench appear in this file. The version format follows a release cycle. A release adds one coherent capability to the workbench. +## 1.12.0 - 2026-08-04 + +### Added + +- A failure-first review queue for unlabeled runs. +- Status filters for the review page, API, and CLI. +- Ordered failed-span summaries in review queue results. +- Deterministic tests for priority ordering, filters, API output, CLI output, and page context. + +### Changed + +- Version numbers moved to 1.12.0. +- The review table shows the first failed span before a reviewer opens the run. + ## 1.11.0 - 2026-08-04 ### Added diff --git a/README.md b/README.md index 5ba1c31..684d36e 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,8 @@ Release 1.10 adds a span detail panel to the error timeline. Click a marker or a Release 1.11 adds ordered failed-span summaries to each day drill-down card and day export. +Release 1.12 adds a failure-first review queue. Filter unlabeled runs by status and read failed-span context before opening a run. + ## Value Agent debugging needs evidence at tool boundaries. @@ -87,7 +89,7 @@ SQLite runs in WAL mode with a busy timeout. Readers keep a committed snapshot. The OpenTelemetry integration stays local by default. Set `ATW_OTEL_CONSOLE=1` to print workbench spans. Set `ATW_OTEL_COLLECTOR_ENDPOINT` to export them to a local collector. -Each stored run keeps a local label and note. They form the review context for long-lived evidence. The server can sweep old evidence on an interval. The scheduler stays off unless you set `ATW_CLEANUP_EVERY_SECONDS`. +Each stored run keeps a local label and note. They form the review context for long-lived evidence. The review queue orders failures first and includes failed-span summaries. The server can sweep old evidence on an interval. The scheduler stays off unless you set `ATW_CLEANUP_EVERY_SECONDS`. ## Setup @@ -207,7 +209,17 @@ The command prints matching run summaries. "run_id": "run-candidate-001", "agent_name": "catalog-assistant", "status": "error", - "tool_count": 3 + "tool_count": 3, + "error_summary": [ + { + "name": "agent.run", + "message": "agent.run ended with status error" + }, + { + "name": "reserve_inventory", + "message": "reservation window expired" + } + ] } ] ``` @@ -1154,6 +1166,17 @@ curl.exe "http://127.0.0.1:8000/api/review" The review page links each run to its annotation form. +The queue shows failed runs first. It sorts each group by start time. Each failed run includes its ordered failed-span summary. + +Filter the queue by status. + +```powershell +python -m agent_trace_workbench.cli review --status error +curl.exe "http://127.0.0.1:8000/api/review?status=error" +``` + +The API and CLI return the same run summary. The summary adds an `error_summary` list for failed spans. + ## Bulk labeling Apply one label to several runs at once. @@ -1497,7 +1520,7 @@ python scripts/check_requirements.py python -m compileall agent_trace_workbench tests ``` -Current verification passes 360 tests, Ruff lint, dependency checks, and Python compilation. CI installs from `requirements-lock.txt` and runs these checks on Python 3.11, 3.12, and 3.13 for every push and pull request. +Current verification passes 365 tests, Ruff lint, dependency checks, and Python compilation. CI installs from `requirements-lock.txt` and runs these checks on Python 3.11, 3.12, and 3.13 for every push and pull request. ## Limitations @@ -1513,6 +1536,8 @@ Labels and notes stay local to the workbench database. Portable export files do The review list shows runs with an empty label. A blank label counts as unreviewed. +The review queue filters by recorded run status. It prioritizes failures, then older runs. + Bulk labeling sets the label only. It leaves the notes on each run untouched. Retention counts from the last ingestion time. A re-ingest resets the clock. @@ -1639,6 +1664,7 @@ The span exporter sends each workbench span as it ends. It does not batch spans. - Release 1.9 complete: add a run-level error timeline to the run detail page. - Release 1.10 complete: add a span detail panel to the run-level error timeline. - Release 1.11 complete: add ordered failed-span summaries to day cards, API output, CLI output, and CSV export. +- Release 1.12 complete: add a failure-first review queue with status filters and failed-span context. - Next: choose the next bounded evidence-review slice. ## Repository map diff --git a/agent_trace_workbench/__init__.py b/agent_trace_workbench/__init__.py index 15d0ceb..14d0da8 100644 --- a/agent_trace_workbench/__init__.py +++ b/agent_trace_workbench/__init__.py @@ -1,3 +1,3 @@ """Agent Trace Workbench package.""" -__version__ = "1.11.0" +__version__ = "1.12.0" diff --git a/agent_trace_workbench/cli.py b/agent_trace_workbench/cli.py index ab605c1..441c9b6 100644 --- a/agent_trace_workbench/cli.py +++ b/agent_trace_workbench/cli.py @@ -130,6 +130,12 @@ def build_parser() -> argparse.ArgumentParser: "review", help="List runs that still need a review label" ) review.add_argument("--limit", type=int, default=20) + review.add_argument( + "--status", + choices=["all", "ok", "error"], + default="all", + help="Show only healthy or failed unlabeled runs", + ) review.add_argument( "--label", default=None, @@ -421,10 +427,12 @@ def main() -> None: else: print(json.dumps(store.list_comparisons(args.limit), indent=2)) elif args.command == "review": + review_status = None if args.status == "all" else args.status if args.label is not None: _validate_annotation("label", args.label) run_ids = args.run_ids or [ - run["run_id"] for run in store.unreviewed_runs(100) + run["run_id"] + for run in store.unreviewed_runs(100, status=review_status) ] if not run_ids: raise SystemExit("No runs to label") @@ -433,7 +441,11 @@ def main() -> None: elif args.run_ids: raise SystemExit("Provide --label together with --run-id") else: - print(json.dumps(store.unreviewed_runs(args.limit), indent=2)) + print( + json.dumps( + store.unreviewed_runs(args.limit, status=review_status), indent=2 + ) + ) elif args.command == "report": if args.older_than_days < 1: raise SystemExit("--older-than must be at least 1 day") diff --git a/agent_trace_workbench/main.py b/agent_trace_workbench/main.py index 3be402f..2105a6d 100644 --- a/agent_trace_workbench/main.py +++ b/agent_trace_workbench/main.py @@ -8,7 +8,7 @@ from datetime import datetime, timedelta, timezone from html import escape from pathlib import Path -from typing import Any +from typing import Any, Literal from urllib.parse import urlencode from fastapi import FastAPI, HTTPException, Query, Request @@ -229,8 +229,9 @@ def compare_page( def review_page( request: Request, limit: int = Query(default=50, ge=1, le=200), + status: Literal["ok", "error"] | None = Query(default=None), ) -> Any: - runs = app.state.store.unreviewed_runs(limit) + runs = app.state.store.unreviewed_runs(limit, status=status) totals = app.state.store.library_report()["totals"] return render_template( request, @@ -239,6 +240,8 @@ def review_page( "runs": runs, "totals": totals, "limit": limit, + "status": status or "", + "failure_count": sum(run["status"] == "error" for run in runs), "store": app.state.store.store_info(), "telemetry": _telemetry_info(), "scheduler": _scheduler_status(app), @@ -304,8 +307,9 @@ def api_runs( @app.get("/api/review") def api_review( limit: int = Query(default=20, ge=1, le=100), + status: Literal["ok", "error"] | None = Query(default=None), ) -> list[dict[str, Any]]: - return app.state.store.unreviewed_runs(limit) + return app.state.store.unreviewed_runs(limit, status=status) @app.post("/api/review/labels") def api_bulk_label(payload: BulkLabelRequest) -> dict[str, Any]: diff --git a/agent_trace_workbench/storage.py b/agent_trace_workbench/storage.py index 666ac9b..f1f1fd2 100644 --- a/agent_trace_workbench/storage.py +++ b/agent_trace_workbench/storage.py @@ -543,23 +543,38 @@ def search_runs(self, query: str, limit: int = 20) -> list[dict[str, Any]]: ).fetchall() return _summarize_runs(connection, rows) - def unreviewed_runs(self, limit: int = 20) -> list[dict[str, Any]]: - """Return run summaries that have no review label. + def unreviewed_runs( + self, + limit: int = 20, + *, + status: str | None = None, + ) -> list[dict[str, Any]]: + """Return unlabeled runs with failure context for review. - A reviewer starts here. Each returned run has an empty label, so - it has not been triaged yet. The review page links each run to its - annotation form. + Failed runs come first. Within each status, older runs come first. + Pass status to show only ok or error runs. """ + if status not in {None, "ok", "error"}: + raise ValueError("status must be 'ok', 'error', or None") safe_limit = max(1, min(limit, 100)) - with traced_operation("storage.unreviewed_runs", {"review.limit": safe_limit}): + attributes: dict[str, Any] = {"review.limit": safe_limit} + if status is not None: + attributes["review.status"] = status + with traced_operation("storage.unreviewed_runs", attributes): with self._connect() as connection: - rows = connection.execute( - "SELECT * FROM runs WHERE label = '' " - "ORDER BY started_at DESC, run_id DESC LIMIT ?", - (safe_limit,), - ).fetchall() - return _summarize_runs(connection, rows) + query = "SELECT * FROM runs WHERE label = ''" + params: list[Any] = [] + if status is not None: + query += " AND status = ?" + params.append(status) + query += ( + " ORDER BY CASE WHEN status = 'error' THEN 0 ELSE 1 END, " + "started_at ASC, run_id ASC LIMIT ?" + ) + params.append(safe_limit) + rows = connection.execute(query, params).fetchall() + return _summarize_runs(connection, rows, include_error_summary=True) def library_report(self, *, older_than_days: int = 30) -> dict[str, Any]: """Return a folder-level summary of the local trace library. diff --git a/pyproject.toml b/pyproject.toml index 721861b..bbcc10c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "agent-trace-workbench" -version = "1.11.0" +version = "1.12.0" description = "A local workbench for recording, replaying, comparing, and inspecting agent traces." readme = "README.md" requires-python = ">=3.11" diff --git a/static/styles.css b/static/styles.css index 66422d9..c490638 100644 --- a/static/styles.css +++ b/static/styles.css @@ -185,6 +185,10 @@ tbody tr:last-child td { border-bottom: 0; } .mono { font: 11px ui-monospace, SFMono-Regular, Consolas, monospace; } .row-changed td { background: rgba(217,95,79,.035); } .row-alert { color: var(--coral); font-weight: 700; } +.review-evidence { display: flex; min-width: 230px; flex-direction: column; gap: 3px; white-space: normal; } +.review-evidence strong { color: var(--coral); font: 800 9px ui-monospace, SFMono-Regular, Consolas, monospace; letter-spacing: .08em; text-transform: uppercase; } +.review-evidence span { overflow: hidden; max-width: 300px; color: var(--navy); font-size: 11px; text-overflow: ellipsis; white-space: nowrap; } +.review-evidence small, .review-clear { color: var(--muted); font: 10px ui-monospace, SFMono-Regular, Consolas, monospace; } .review-section { padding-top: 48px; } .cleanup-section { padding-top: 48px; } .cleanup-section-last { padding-bottom: 88px; } diff --git a/templates/dashboard.html b/templates/dashboard.html index e89e140..81e7214 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -2,7 +2,7 @@ {% block title %}Runs | Agent Trace Workbench{% endblock %} {% block content %}
-
LOCAL OBSERVABILITY / RELEASE 1.9
+
LOCAL OBSERVABILITY / RELEASE 1.12

See what your agent did.
Replay why it did it.

Ingest local JSON traces, import OTLP exports, watch a folder, inspect every tool boundary, replay recorded work, compare runs, label evidence for review, and read a folder-level library report without a hosted service.

@@ -42,7 +42,7 @@

See what your agent did.
Replay why it did it.

AInspectTimeline, attributes, and failures↗ BReplayStable handlers or recorded fallback↗ CCompareCalls, timing, results, and outcomes↗ - DReviewTriage runs that still need a label↗ + DReviewFailures first with failed-span context↗ EReportFolder-level summary of the library↗
FastAPI SQLite OpenTelemetry Local collector
diff --git a/templates/review.html b/templates/review.html index a070168..24ab1ee 100644 --- a/templates/review.html +++ b/templates/review.html @@ -3,29 +3,34 @@ {% block content %}
-
REVIEW LIST / UNLABELED EVIDENCE
-

Give every run a label.

-

Runs without a label have not been triaged yet. Open one, note what it showed, and it leaves this list.

+
REVIEW QUEUE / FAILURE-FIRST
+

Start with the runs that need answers.

+

Failed runs appear first. Each row shows its first failed span. Label a run after you inspect it.

- JSON + JSON Library report
Needs review{{ runs|length }} shownin this view
+
Failures shown{{ failure_count }}priority evidence
Unlabeled total{{ totals.unlabeled_runs }}across the library
-
Failure runs{{ totals.failure_runs }}unlabeled or not
-
REVIEW QUEUE

Unlabeled runs

{{ totals.labeled_runs }} labeled ยท {{ totals.unlabeled_runs }} left
+
REVIEW QUEUE

{% if status == 'error' %}Failed{% elif status == 'ok' %}Healthy{% else %}Unlabeled{% endif %} runs

{{ runs|length }} shown · failures first
+
+ All + Failures + Healthy +
{% if runs %}
-
{% for run in runs %}{% endfor %}
StatusAgentRunTool callsDurationSourceActions
{{ run.status }}{{ run.agent_name }}{{ run.run_id }}{{ run.tool_count }}{{ run.duration_ms|round(1) }} ms{{ run.source_dir or 'api' }}/{{ run.source_name }}Review
+
{% for run in runs %}{% endfor %}
StatusAgentRunEvidenceTool callsDurationSourceActions
{{ run.status }}{{ run.agent_name }}{{ run.run_id }}{% if run.error_summary %}
{{ run.error_summary|length }} failed span{% if run.error_summary|length != 1 %}s{% endif %}{{ run.error_summary[0].name }} · {{ run.error_summary[0].message }}{% if run.error_summary|length > 1 %}+ {{ run.error_summary|length - 1 }} more{% endif %}
{% else %}No failed spans{% endif %}		{{ run.tool_count }}{{ run.duration_ms|round(1) }} ms{{ run.source_dir or 'api' }}/{{ run.source_name }}Review
{% else %}
✓

Nothing left to review

Every run in the library has a label. Add one to a run to clear its review flag.

{% endif %} diff --git a/tests/test_review_report.py b/tests/test_review_report.py index d6533a6..f44000d 100644 --- a/tests/test_review_report.py +++ b/tests/test_review_report.py @@ -43,6 +43,41 @@ def test_store_unreviewed_runs_excludes_labeled(tmp_path, baseline, candidate): assert reviewed[0]["tool_count"] == 3 +def test_store_review_queue_prioritizes_failures_and_includes_context( + tmp_path, baseline, candidate +): + store = TraceStore(tmp_path / "review.db") + store.ingest(baseline, "baseline.json") + store.ingest(candidate, "candidate.json") + + reviewed = store.unreviewed_runs() + + assert [run["run_id"] for run in reviewed] == [ + "run-candidate-001", + "run-baseline-001", + ] + assert [item["name"] for item in reviewed[0]["error_summary"]] == [ + "agent.run", + "reserve_inventory", + ] + assert reviewed[1]["error_summary"] == [] + + +def test_store_review_queue_filters_by_status(tmp_path, baseline, candidate): + store = TraceStore(tmp_path / "review.db") + store.ingest(baseline, "baseline.json") + store.ingest(candidate, "candidate.json") + + assert [run["run_id"] for run in store.unreviewed_runs(status="error")] == [ + "run-candidate-001" + ] + assert [run["run_id"] for run in store.unreviewed_runs(status="ok")] == [ + "run-baseline-001" + ] + with pytest.raises(ValueError, match="status"): + store.unreviewed_runs(status="unset") + + def test_store_unreviewed_runs_returns_empty_when_all_labeled(tmp_path, baseline): store = TraceStore(tmp_path / "review.db") store.ingest(baseline, "baseline.json") @@ -153,6 +188,21 @@ def test_api_lists_review_runs(tmp_path, baseline, candidate): } +def test_api_filters_review_queue_and_returns_failure_context(tmp_path, baseline, candidate): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=candidate.as_jsonable()) + + response = client.get("/api/review", params={"status": "error"}) + + assert response.status_code == 200 + assert [run["run_id"] for run in response.json()] == ["run-candidate-001"] + assert response.json()[0]["error_summary"][1]["message"] == ( + "reservation window expired" + ) + assert client.get("/api/review", params={"status": "unset"}).status_code == 422 + + def test_api_returns_library_report(tmp_path, baseline, candidate): client = TestClient(create_app(tmp_path / "api.db")) client.post("/api/traces", json=baseline.as_jsonable()) @@ -178,6 +228,20 @@ def test_review_page_shows_unlabeled_runs(tmp_path, baseline, candidate): assert "run-baseline-001" not in page +def test_review_page_shows_failure_context_and_status_filters(tmp_path, baseline, candidate): + client = TestClient(create_app(tmp_path / "api.db")) + client.post("/api/traces", json=baseline.as_jsonable()) + client.post("/api/traces", json=candidate.as_jsonable()) + + page = client.get("/review", params={"status": "error"}).text + + assert "Failed runs" in page + assert "agent.run" in page + assert "agent.run ended with status error" in page + assert "status=error" in page + assert "run-baseline-001" not in page + + def test_report_page_shows_totals_and_folder_rows(tmp_path, baseline, candidate): client = TestClient(create_app(tmp_path / "api.db")) client.post("/api/traces", json=baseline.as_jsonable()) @@ -206,6 +270,21 @@ def test_cli_review_lists_unlabeled_runs(tmp_path, baseline, candidate, monkeypa assert [run["run_id"] for run in results] == ["run-candidate-001"] +def test_cli_review_filters_failed_runs(tmp_path, baseline, candidate, monkeypatch, capsys): + store = TraceStore(tmp_path / "cli.db") + store.ingest(baseline, "baseline.json") + store.ingest(candidate, "candidate.json") + + monkeypatch.setattr( + "sys.argv", ["atw", "--db", str(tmp_path / "cli.db"), "review", "--status", "error"] + ) + main() + + results = json.loads(capsys.readouterr().out) + assert [run["run_id"] for run in results] == ["run-candidate-001"] + assert results[0]["error_summary"][0]["name"] == "agent.run" + + def test_cli_report_prints_library_summary(tmp_path, baseline, candidate, monkeypatch, capsys): store = TraceStore(tmp_path / "cli.db") store.ingest(baseline, "baseline.json")