diff --git a/tests/test_consolidated_report.py b/tests/test_consolidated_report.py index d287e4c..caf12ff 100644 --- a/tests/test_consolidated_report.py +++ b/tests/test_consolidated_report.py @@ -1,6 +1,10 @@ from __future__ import annotations -from zwill.consolidated_report import downloads_section_html, render_consolidated_report +from zwill.consolidated_report import ( + downloads_section_html, + mark_intermediate_page_html, + render_consolidated_report, +) from zwill.reporting import EP_REPORT_CSS @@ -40,6 +44,22 @@ def test_composes_pages_into_one_document_with_toc() -> None: assert 'href="audit/run.html"' in html and "Twin run audit" in html +def test_section_banner_is_added_outside_main_and_excluded_from_report() -> None: + page = _page("", "
BODY-CONTENT
") + marked = mark_intermediate_page_html(page) + # Banner is inserted (after , before
). + assert "This is one section of a larger report" in marked + assert marked.index("This is one section") < marked.index(" is). + html = render_consolidated_report( + survey="demo", sections=[("s", "Section", marked)], downloads_section="" + ) + assert "BODY-CONTENT" in html + assert "This is one section of a larger report" not in html + + def test_empty_sections_and_downloads_are_skipped() -> None: html = render_consolidated_report( survey="demo", diff --git a/tests/test_report_conditional_baseline.py b/tests/test_report_conditional_baseline.py index 7a66c47..608fe3a 100644 --- a/tests/test_report_conditional_baseline.py +++ b/tests/test_report_conditional_baseline.py @@ -94,6 +94,38 @@ def test_executive_summary_features_conditional_baseline(tmp_path) -> None: assert comp["share_twin_better"] == 1.0 +def test_stale_narrative_banner_when_model_set_changed(tmp_path) -> None: + twin = [_row("openai:gpt-5.5", "r1", "A", 0.20, 1), _row("openai:gpt-5.5", "r2", "B", 0.25, 1)] + build_executive_summary( + twin, + survey="demo", + path=tmp_path / "exec.html", + markdown_path=None, + simulations=25, + seed=1, + generated_markdown="The GPT-5.5 twin captures individual signal.", + generation={"report_id": "r", "stale": True, "stale_reason": "written for openai:gpt-5.5; current: anthropic:claude-opus-4-8, openai:gpt-5.5"}, + ) + html = (tmp_path / "exec.html").read_text() + assert "This written summary is out of date" in html + assert "anthropic:claude-opus-4-8" in html + + +def test_no_stale_banner_for_fresh_narrative(tmp_path) -> None: + twin = [_row("openai:gpt-5.5", "r1", "A", 0.20, 1), _row("openai:gpt-5.5", "r2", "B", 0.25, 1)] + build_executive_summary( + twin, + survey="demo", + path=tmp_path / "exec.html", + markdown_path=None, + simulations=25, + seed=1, + generated_markdown="Fresh narrative.", + generation={"report_id": "r"}, + ) + assert "This written summary is out of date" not in (tmp_path / "exec.html").read_text() + + def test_executive_summary_unchanged_without_baseline(tmp_path) -> None: twin = [_row("openai:gpt-5.5", "r1", "A", 0.20, 1), _row("openai:gpt-5.5", "r2", "B", 0.25, 1)] result = build_executive_summary( diff --git a/zwill/consolidated_report.py b/zwill/consolidated_report.py index 89f87a2..71dfdf3 100644 --- a/zwill/consolidated_report.py +++ b/zwill/consolidated_report.py @@ -17,6 +17,31 @@ _STYLE_RE = re.compile(r"", re.S) _MAIN_RE = re.compile(r"]*>(.*)
", re.S) +_BODY_RE = re.compile(r"(]*>)", re.I) + +_SECTION_BANNER_MARK = "This is one section of a larger report" +_SECTION_BANNER = ( + "
" + _SECTION_BANNER_MARK + ". " + "Open the full report for all sections, the table of contents, " + "and the executive summary.
" +) + + +def mark_intermediate_page_html(html: str, index_href: str = "index.html") -> str: + """Insert a 'this is a section' banner after ````. + + Placed outside ``
`` so :func:`render_consolidated_report` (which lifts + only the ``
`` body) never pulls it into the combined report. Idempotent. + """ + if _SECTION_BANNER_MARK in html: + return html + match = _BODY_RE.search(html) + if not match: + return html + banner = _SECTION_BANNER.format(index_href=escape(index_href)) + return html[: match.end()] + "\n" + banner + html[match.end() :] _CONSOLIDATED_CSS = """ .report-shell { max-width: 1180px; margin: 0 auto; padding: 0 16px 4rem; } diff --git a/zwill/executive_summary.py b/zwill/executive_summary.py index 1320cea..575c101 100644 --- a/zwill/executive_summary.py +++ b/zwill/executive_summary.py @@ -435,14 +435,24 @@ def render_html( for row in individual.get("per_question", []) ) generation_note = "" + stale_banner = "" if generation: generation_note = ( f"

Generated analysis: {escape(str(generation.get('report_id') or ''))}" f"{' · model: ' + escape(str(generation.get('model'))) + '' if generation.get('model') else ''}

" ) + if generation.get("stale"): + reason = escape(str(generation.get("stale_reason") or "")) + stale_banner = ( + "
" + "⚠ This written summary is out of date. It was generated for a different set of " + f"models than the tables below now show{f' ({reason})' if reason else ''}. Regenerate it with " + "zwill twin-results executive-summary-export → run → import, then rebuild the report.
" + ) if generated_markdown: generated_body = markdown_to_html(remove_leading_executive_summary_heading(generated_markdown)) executive_body = ( + f"{stale_banner}" "
" "

Executive Summary

" f"{generated_body}" diff --git a/zwill/generated_reports.py b/zwill/generated_reports.py index b485369..30edfb2 100644 --- a/zwill/generated_reports.py +++ b/zwill/generated_reports.py @@ -348,6 +348,16 @@ def build_executive_summary_report_context( "job_ids": job_ids, "model": getattr(args, "prediction_model", None), "questions": heldout_names, + # Fingerprint of the twin model set this narrative describes, so a + # rebuild can detect when the prose predates an added/removed model. + # Baseline rows are excluded so the fingerprint compares twin-to-twin. + "model_labels": sorted( + { + str(row.get("model_label")) + for row in rows + if row.get("model_label") and not str(row.get("model_label")).startswith("baseline:") + } + ), }, "heldout_questions": [ { diff --git a/zwill/report_bundle.py b/zwill/report_bundle.py index c9b0c9b..a55bb9a 100644 --- a/zwill/report_bundle.py +++ b/zwill/report_bundle.py @@ -1,7 +1,11 @@ from __future__ import annotations from .cli import * # noqa: F403 -from .consolidated_report import downloads_section_html, render_consolidated_report +from .consolidated_report import ( + downloads_section_html, + mark_intermediate_page_html, + render_consolidated_report, +) from .twin_baseline import MODEL_LABEL as BASELINE_MODEL_LABEL # Bundle pages that fold into the single scrollable report as sections, in order. @@ -1046,6 +1050,24 @@ def add_page(page: dict[str, Any]) -> None: model=getattr(args, "model", None), questions=heldout_questions_for_report, ) + # Flag a cached narrative that predates the current model set: the prose + # is matched on jobs+questions but not on the models inside them, so an + # added/removed model can leave stale prose over fresh tables. + if generated_executive: + current_models = sorted( + { + str(row.get("model_label")) + for row in twin_rows + if row.get("model_label") and not str(row.get("model_label")).startswith("baseline:") + } + ) + narrative_models = generated_executive["generation"].get("narrative_model_labels") + if narrative_models is not None and sorted(narrative_models) != current_models: + generated_executive["generation"]["stale"] = True + generated_executive["generation"]["stale_reason"] = ( + f"written for {', '.join(narrative_models) or 'no models'}; " + f"current: {', '.join(current_models) or 'no models'}" + ) executive_result = build_executive_summary( twin_rows, survey=survey, @@ -1255,12 +1277,24 @@ def add_page(page: dict[str, Any]) -> None: # it replaces the old digest index. When there is no ready twin validation # yet, fall back to the readiness/next-steps index so the bundle is still # navigable. - consolidated_report = build_consolidated_report_html( - output_dir, pages, survey, (manifest.get("executive_summary") or {}).get("path") - ) + executive_html_path = (manifest.get("executive_summary") or {}).get("path") + consolidated_report = build_consolidated_report_html(output_dir, pages, survey, executive_html_path) if consolidated_report: (output_dir / "report.html").write_text(consolidated_report) (output_dir / "index.html").write_text(consolidated_report) + # Mark the folded-in pages as sections so they are not mistaken for the + # report when opened on their own (banner sits outside
, so it is + # never pulled into the consolidated report above). + mark_paths = [executive_html_path] if executive_html_path else [] + by_id = {page.get("page_id"): page for page in pages} + for page_id in ("twin-validation", "survey-profile", "one-shot-marginals"): + page = by_id.get(page_id) + if page and page.get("path"): + mark_paths.append(page["path"]) + for mark_path in mark_paths: + page_file = Path(mark_path) + if page_file.exists(): + page_file.write_text(mark_intermediate_page_html(page_file.read_text())) else: (output_dir / "index.html").write_text(render_report_bundle_index(manifest)) write_bundle_json(manifest_path, manifest) @@ -1358,6 +1392,9 @@ def find_imported_executive_summary_report( "markdown_path": str(markdown_path), "import_path": str(import_path), "imported_at": imported.get("imported_at"), + # The model set this narrative was written against (None for narratives + # generated before fingerprinting existed). + "narrative_model_labels": filters.get("model_labels"), } candidates.append( {