From e28ef5a506fe8f41c9acf83d744bf6762de27772 Mon Sep 17 00:00:00 2001 From: Jordak Date: Mon, 8 Jun 2026 16:12:28 -0700 Subject: [PATCH] Add per-trial provenance to capability digests --- agentlab/reports/capability_digest.py | 20 +++++ agentlab/reports/capability_digest_html.py | 22 +++++- tests/test_evidence.py | 89 ++++++++++++++++++++++ 3 files changed, 130 insertions(+), 1 deletion(-) diff --git a/agentlab/reports/capability_digest.py b/agentlab/reports/capability_digest.py index d132ce1..ed6555c 100644 --- a/agentlab/reports/capability_digest.py +++ b/agentlab/reports/capability_digest.py @@ -171,6 +171,7 @@ def _run_context_lines(context: MarkdownRunContext) -> List[str]: "Cost USD", "Duration ms", "Scope Oracle", + "Task Provenance", ], [_trial_row(result) for result in context.results], ) @@ -450,9 +451,23 @@ def _trial_row(result: OutcomeEvidence) -> List[object]: _unknown_if_none(result.cost_usd), result.duration_ms, compact_scope_oracle_metadata(result.scope_oracle), + _trial_provenance(result), ] +def _trial_provenance(result: OutcomeEvidence) -> str: + return "; ".join( + [ + f"repo={_display_unknown(result.task_repo)}", + f"commit={_display_unknown(result.task_commit)}", + ( + "workspace_base_ref=" + f"{_format_run_surface_value(result.run_surface.get('workspace_base_ref'))}" + ), + ] + ) + + def _markdown_table(headers: List[str], rows: List[List[object]]) -> List[str]: table = [ "| " + " | ".join(_escape_cell(header) for header in headers) + " |", @@ -508,6 +523,11 @@ def _unknown_if_none(value: object) -> object: return value +def _display_unknown(value: object) -> str: + text = "" if value is None else str(value) + return text or "unknown" + + def _escape_cell(value: object) -> str: text = "" if value is None else str(value) return text.replace("|", "\\|").replace("\n", "
") diff --git a/agentlab/reports/capability_digest_html.py b/agentlab/reports/capability_digest_html.py index 72a41ae..36a3859 100644 --- a/agentlab/reports/capability_digest_html.py +++ b/agentlab/reports/capability_digest_html.py @@ -69,6 +69,7 @@ "Secondary Review Labels", "Primary Review Label", "Exclusions", + "Task Provenance", "Scope Oracle", "Operability Dimension", "Evidence", @@ -500,7 +501,7 @@ def _context_page( {_summary_section("Outcome Summary", _outcome_rows(context, render_context), ["Task", "Type", "Total", "Fair", "Excluded", "Passes", "Accepted", "Pass Rate", "pass@k", "pass^k"])} {_summary_section("Token Summary", _token_rows(context, render_context), ["Task", "Type", "IO Tokens", "Cached Tokens", "Reason Tokens", "IO Tok / Verified", "IO Tok / Accepted", "Cached Tok / Verified", "Reason Tok / Verified"])} {_summary_section("Review and Patch Summary", _review_rows(context, render_context), ["Task", "Type", "Median ms", "Median Files", "Median +Lines", "Median -Lines", "Primary Review Labels", "Secondary Review Labels", "Exclusions"], note=_review_patch_size_caveat_note(context))} - {_summary_section("Trial Evidence", _trial_rows(context, render_context), ["Task", "Type", "Trial", "Grader Outcome", "Validity", "Primary Review Label", "Secondary Review Labels", "Exclusion", "Files", "+Lines", "-Lines", "Input Tokens", "Cached Input Tokens", "Output Tokens", "Reasoning Tokens", "Cost USD", "Duration ms", "Scope Oracle", "Report", "Transcript", "Diff", "Result"], note=_trial_evidence_note(context.results))} + {_summary_section("Trial Evidence", _trial_rows(context, render_context), ["Task", "Type", "Trial", "Grader Outcome", "Validity", "Primary Review Label", "Secondary Review Labels", "Exclusion", "Files", "+Lines", "-Lines", "Input Tokens", "Cached Input Tokens", "Output Tokens", "Reasoning Tokens", "Cost USD", "Duration ms", "Scope Oracle", "Task Provenance", "Report", "Transcript", "Diff", "Result"], note=_trial_evidence_note(context.results))} """ @@ -721,6 +722,7 @@ def _trial_rows( "Task": _task_link_from_result(result, render_context), "Type": _text(result.eval_type), "Trial": _text(result.trial_id), + "Task Provenance": _trial_provenance(result), "Grader Outcome": _text(result.status), "Validity": _text(result.trial_validity), "Primary Review Label": _text(result.primary_review_label), @@ -755,6 +757,19 @@ def _trial_rows( ] +def _trial_provenance(result: OutcomeEvidence) -> str: + return "
".join( + [ + _text(f"repo={_display_unknown(result.task_repo)}"), + _text(f"commit={_display_unknown(result.task_commit)}"), + _text( + "workspace_base_ref=" + f"{_format_run_surface_value(result.run_surface.get('workspace_base_ref'))}" + ), + ] + ) + + def _patch_size_caveat_note(results: Sequence[OutcomeEvidence]) -> str: if not has_patch_size_caveats(results): return "" @@ -1072,6 +1087,11 @@ def _unknown_if_none(value: object) -> object: return value +def _display_unknown(value: object) -> str: + text = "" if value is None else str(value) + return text or "unknown" + + def _pill(text: str, kind: str = "") -> str: return f'{_text(text)}' diff --git a/tests/test_evidence.py b/tests/test_evidence.py index bf4f148..7fd1ab8 100644 --- a/tests/test_evidence.py +++ b/tests/test_evidence.py @@ -199,6 +199,95 @@ def test_digest_renders_scope_oracle_metadata(self): self.assertIn("allowed_paths=src/", html_report) self.assertIn("forbidden_paths=src/private/", html_report) + def test_digest_maps_trial_provenance_for_mixed_workspace_bases(self): + results = [ + normalize_outcome_evidence( + { + "trial_id": "trial-a", + "task_id": "task-a", + "task_repo": "https://github.com/example/repo-a", + "task_commit": "aaa111", + "eval_suite": "starter", + "eval_type": "regression", + "agent_name": "codex", + "model_name": "model-a", + "agent_harness_config": {"reasoning_effort": "xhigh"}, + "run_surface": {"workspace_base_ref": "refs/base-a"}, + "status": "passed", + "success": True, + "duration_ms": 100, + "files_changed": ["app.py"], + "lines_added": 5, + "lines_deleted": 1, + "run_dir": "runs/trial-a", + } + ), + normalize_outcome_evidence( + { + "trial_id": "trial-b", + "task_id": "task-b", + "task_repo": "https://github.com/example/repo-b", + "task_commit": "bbb222", + "eval_suite": "starter", + "eval_type": "regression", + "agent_name": "codex", + "model_name": "model-a", + "agent_harness_config": {"reasoning_effort": "xhigh"}, + "run_surface": {"workspace_base_ref": "refs/base-b"}, + "status": "failed", + "success": False, + "duration_ms": 200, + "files_changed": ["app.py"], + "lines_added": 3, + "lines_deleted": 0, + "run_dir": "runs/trial-b", + } + ), + ] + + digest = render_capability_evidence_digest(results) + html_report = render_capability_evidence_digest_html(results) + + self.assertIn("Workspace Base Ref", digest) + self.assertIn("mixed: refs/base-a; refs/base-b", digest) + self.assertIn( + "| Task | Type | Trial | Grader Outcome | Validity |", + digest, + ) + self.assertIn("Task Provenance", digest) + self.assertRegex( + digest, + ( + r"\| task-a \| regression \| trial-a \| passed \| valid \|.*" + r"repo=https://github\.com/example/repo-a; " + r"commit=aaa111; workspace_base_ref=refs/base-a \|" + ), + ) + self.assertRegex( + digest, + ( + r"\| task-b \| regression \| trial-b \| failed \| valid \|.*" + r"repo=https://github\.com/example/repo-b; " + r"commit=bbb222; workspace_base_ref=refs/base-b \|" + ), + ) + self.assertIn("Task Provenance", html_report) + self.assertIn("mixed: refs/base-a; refs/base-b", html_report) + self.assertIn( + ( + "repo=https://github.com/example/repo-a
" + "commit=aaa111
workspace_base_ref=refs/base-a" + ), + html_report, + ) + self.assertIn( + ( + "repo=https://github.com/example/repo-b
" + "commit=bbb222
workspace_base_ref=refs/base-b" + ), + html_report, + ) + def test_nested_setup_created_untracked_patch_size_caveat_round_trips(self): result = normalize_outcome_evidence( {