Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions agentlab/reports/capability_digest.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ def _run_context_lines(context: MarkdownRunContext) -> List[str]:
"Cost USD",
"Duration ms",
"Scope Oracle",
"Task Provenance",
],
[_trial_row(result) for result in context.results],
)
Expand Down Expand Up @@ -450,9 +451,23 @@ def _trial_row(result: OutcomeEvidence) -> List[object]:
_unknown_if_none(result.cost_usd),
result.duration_ms,
compact_scope_oracle_metadata(result.scope_oracle),
_trial_provenance(result),
]


def _trial_provenance(result: OutcomeEvidence) -> str:
return "; ".join(
[
f"repo={_display_unknown(result.task_repo)}",
f"commit={_display_unknown(result.task_commit)}",
(
"workspace_base_ref="
f"{_format_run_surface_value(result.run_surface.get('workspace_base_ref'))}"
),
]
)


def _markdown_table(headers: List[str], rows: List[List[object]]) -> List[str]:
table = [
"| " + " | ".join(_escape_cell(header) for header in headers) + " |",
Expand Down Expand Up @@ -508,6 +523,11 @@ def _unknown_if_none(value: object) -> object:
return value


def _display_unknown(value: object) -> str:
text = "" if value is None else str(value)
return text or "unknown"


def _escape_cell(value: object) -> str:
text = "" if value is None else str(value)
return text.replace("|", "\\|").replace("\n", "<br>")
22 changes: 21 additions & 1 deletion agentlab/reports/capability_digest_html.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
"Secondary Review Labels",
"Primary Review Label",
"Exclusions",
"Task Provenance",
"Scope Oracle",
"Operability Dimension",
"Evidence",
Expand Down Expand Up @@ -500,7 +501,7 @@ def _context_page(
{_summary_section("Outcome Summary", _outcome_rows(context, render_context), ["Task", "Type", "Total", "Fair", "Excluded", "Passes", "Accepted", "Pass Rate", "pass@k", "pass^k"])}
{_summary_section("Token Summary", _token_rows(context, render_context), ["Task", "Type", "IO Tokens", "Cached Tokens", "Reason Tokens", "IO Tok / Verified", "IO Tok / Accepted", "Cached Tok / Verified", "Reason Tok / Verified"])}
{_summary_section("Review and Patch Summary", _review_rows(context, render_context), ["Task", "Type", "Median ms", "Median Files", "Median +Lines", "Median -Lines", "Primary Review Labels", "Secondary Review Labels", "Exclusions"], note=_review_patch_size_caveat_note(context))}
{_summary_section("Trial Evidence", _trial_rows(context, render_context), ["Task", "Type", "Trial", "Grader Outcome", "Validity", "Primary Review Label", "Secondary Review Labels", "Exclusion", "Files", "+Lines", "-Lines", "Input Tokens", "Cached Input Tokens", "Output Tokens", "Reasoning Tokens", "Cost USD", "Duration ms", "Scope Oracle", "Report", "Transcript", "Diff", "Result"], note=_trial_evidence_note(context.results))}
{_summary_section("Trial Evidence", _trial_rows(context, render_context), ["Task", "Type", "Trial", "Grader Outcome", "Validity", "Primary Review Label", "Secondary Review Labels", "Exclusion", "Files", "+Lines", "-Lines", "Input Tokens", "Cached Input Tokens", "Output Tokens", "Reasoning Tokens", "Cost USD", "Duration ms", "Scope Oracle", "Task Provenance", "Report", "Transcript", "Diff", "Result"], note=_trial_evidence_note(context.results))}
</main>
"""

Expand Down Expand Up @@ -721,6 +722,7 @@ def _trial_rows(
"Task": _task_link_from_result(result, render_context),
"Type": _text(result.eval_type),
"Trial": _text(result.trial_id),
"Task Provenance": _trial_provenance(result),
"Grader Outcome": _text(result.status),
"Validity": _text(result.trial_validity),
"Primary Review Label": _text(result.primary_review_label),
Expand Down Expand Up @@ -755,6 +757,19 @@ def _trial_rows(
]


def _trial_provenance(result: OutcomeEvidence) -> str:
return "<br>".join(
[
_text(f"repo={_display_unknown(result.task_repo)}"),
_text(f"commit={_display_unknown(result.task_commit)}"),
_text(
"workspace_base_ref="
f"{_format_run_surface_value(result.run_surface.get('workspace_base_ref'))}"
),
]
)


def _patch_size_caveat_note(results: Sequence[OutcomeEvidence]) -> str:
if not has_patch_size_caveats(results):
return ""
Expand Down Expand Up @@ -1072,6 +1087,11 @@ def _unknown_if_none(value: object) -> object:
return value


def _display_unknown(value: object) -> str:
text = "" if value is None else str(value)
return text or "unknown"


def _pill(text: str, kind: str = "") -> str:
return f'<span class="pill {kind}">{_text(text)}</span>'

Expand Down
89 changes: 89 additions & 0 deletions tests/test_evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,95 @@ def test_digest_renders_scope_oracle_metadata(self):
self.assertIn("allowed_paths=src/", html_report)
self.assertIn("forbidden_paths=src/private/", html_report)

def test_digest_maps_trial_provenance_for_mixed_workspace_bases(self):
results = [
normalize_outcome_evidence(
{
"trial_id": "trial-a",
"task_id": "task-a",
"task_repo": "https://github.com/example/repo-a",
"task_commit": "aaa111",
"eval_suite": "starter",
"eval_type": "regression",
"agent_name": "codex",
"model_name": "model-a",
"agent_harness_config": {"reasoning_effort": "xhigh"},
"run_surface": {"workspace_base_ref": "refs/base-a"},
"status": "passed",
"success": True,
"duration_ms": 100,
"files_changed": ["app.py"],
"lines_added": 5,
"lines_deleted": 1,
"run_dir": "runs/trial-a",
}
),
normalize_outcome_evidence(
{
"trial_id": "trial-b",
"task_id": "task-b",
"task_repo": "https://github.com/example/repo-b",
"task_commit": "bbb222",
"eval_suite": "starter",
"eval_type": "regression",
"agent_name": "codex",
"model_name": "model-a",
"agent_harness_config": {"reasoning_effort": "xhigh"},
"run_surface": {"workspace_base_ref": "refs/base-b"},
"status": "failed",
"success": False,
"duration_ms": 200,
"files_changed": ["app.py"],
"lines_added": 3,
"lines_deleted": 0,
"run_dir": "runs/trial-b",
}
),
]

digest = render_capability_evidence_digest(results)
html_report = render_capability_evidence_digest_html(results)

self.assertIn("Workspace Base Ref", digest)
self.assertIn("mixed: refs/base-a; refs/base-b", digest)
self.assertIn(
"| Task | Type | Trial | Grader Outcome | Validity |",
digest,
)
self.assertIn("Task Provenance", digest)
self.assertRegex(
digest,
(
r"\| task-a \| regression \| trial-a \| passed \| valid \|.*"
r"repo=https://github\.com/example/repo-a; "
r"commit=aaa111; workspace_base_ref=refs/base-a \|"
),
)
self.assertRegex(
digest,
(
r"\| task-b \| regression \| trial-b \| failed \| valid \|.*"
r"repo=https://github\.com/example/repo-b; "
r"commit=bbb222; workspace_base_ref=refs/base-b \|"
),
)
self.assertIn("Task Provenance", html_report)
self.assertIn("mixed: refs/base-a; refs/base-b", html_report)
self.assertIn(
(
"repo=https://github.com/example/repo-a<br>"
"commit=aaa111<br>workspace_base_ref=refs/base-a"
),
html_report,
)
self.assertIn(
(
"repo=https://github.com/example/repo-b<br>"
"commit=bbb222<br>workspace_base_ref=refs/base-b"
),
html_report,
)

def test_nested_setup_created_untracked_patch_size_caveat_round_trips(self):
result = normalize_outcome_evidence(
{
Expand Down