From ed587e5546adefcc5f6508b05ef3d9ea053c1c03 Mon Sep 17 00:00:00 2001 From: John Horton Date: Thu, 9 Jul 2026 09:34:52 -0400 Subject: [PATCH] Feature the conditional baseline on the executive summary and index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two headline pages a decision-maker actually reads — the executive summary and the bundle index — compared the twin only against uniform guessing, while the one comparison that matters (twin vs the deployable XGBoost conditional baseline) was buried on the technical validation page. So the headline story was weaker and different from the detail page. build_executive_summary now takes the baseline rows and computes a twin-vs- conditional-baseline comparison (aggregate baseline metrics + a paired win rate and p(actual) delta over the rows both scored). The executive summary's Main Evidence table gains a "Conditional baseline (XGBoost)" column and a lead sentence framing it as the deployable bar (uniform/empirical demoted to context), and the comparison is returned so the bundle index shows a "Beats conditional baseline / p(actual) vs baseline" row next to the twin's p(actual). All gated on a baseline being present, so reports without one are byte-for-byte unchanged. The report bundle passes the baseline rows folded in by #73; the standalone `twin-results executive-summary` command is unchanged for now. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_report_conditional_baseline.py | 41 +++++++++++++-- zwill/executive_summary.py | 64 ++++++++++++++++++++++- zwill/report_bundle.py | 11 ++++ 3 files changed, 112 insertions(+), 4 deletions(-) diff --git a/tests/test_report_conditional_baseline.py b/tests/test_report_conditional_baseline.py index 8032ef1..7a66c47 100644 --- a/tests/test_report_conditional_baseline.py +++ b/tests/test_report_conditional_baseline.py @@ -3,12 +3,14 @@ import math from zwill.cli import build_twin_report +from zwill.executive_summary import build_executive_summary from zwill.reporting import render_twin_summary_report_html from zwill.twin_baseline import MODEL_LABEL as BASELINE_MODEL_LABEL -def _row(model_label: str, respondent: str, actual: str, nll: float, correct: int) -> dict: - probs = {"A": 0.8, "B": 0.2} if actual == "A" else {"A": 0.2, "B": 0.8} +def _row(model_label: str, respondent: str, actual: str, nll: float, correct: int, p_actual: float = 0.8) -> dict: + other = "B" if actual == "A" else "A" + probs = {actual: p_actual, other: 1.0 - p_actual} return { "job_id": "twin1" if model_label != BASELINE_MODEL_LABEL else "base1", "survey": "demo", @@ -22,7 +24,7 @@ def _row(model_label: str, respondent: str, actual: str, nll: float, correct: in "option_labels": ["A", "B"], "probabilities": probs, "raw_probabilities": [probs["A"], probs["B"]], - "probability_actual": probs[actual], + "probability_actual": p_actual, "uniform_probability_actual": 0.5, "uniform_negative_log_likelihood": math.log(2), "negative_log_likelihood": nll, @@ -67,3 +69,36 @@ def test_no_conditional_column_when_baseline_absent() -> None: html = _render(twin) assert "NLL improvement vs conditional baseline" not in html assert "Conditional baseline (XGBoost)" not in html + + +def test_executive_summary_features_conditional_baseline(tmp_path) -> None: + twin = [_row("openai:gpt-5.5", "r1", "A", 0.20, 1, p_actual=0.8), _row("openai:gpt-5.5", "r2", "B", 0.25, 1, p_actual=0.8)] + baseline = [_row(BASELINE_MODEL_LABEL, "r1", "A", 0.55, 1, p_actual=0.5), _row(BASELINE_MODEL_LABEL, "r2", "B", 0.60, 0, p_actual=0.5)] + result = build_executive_summary( + twin, + survey="demo", + path=tmp_path / "exec.html", + markdown_path=None, + simulations=25, + seed=1, + baseline_rows=baseline, + ) + html = (tmp_path / "exec.html").read_text() + # Main Evidence gains the conditional baseline as a column + a lead sentence. + assert "Conditional baseline (XGBoost)" in html + assert "Versus the deployable bar" in html + # The comparison is exposed for the index tile. + comp = result["conditional_comparison"] + assert comp is not None and comp["matched_rows"] == 2 + # Twin (p_actual 0.8/0.8) clearly beats baseline here. + assert comp["share_twin_better"] == 1.0 + + +def test_executive_summary_unchanged_without_baseline(tmp_path) -> None: + twin = [_row("openai:gpt-5.5", "r1", "A", 0.20, 1), _row("openai:gpt-5.5", "r2", "B", 0.25, 1)] + result = build_executive_summary( + twin, survey="demo", path=tmp_path / "exec.html", markdown_path=None, simulations=25, seed=1 + ) + html = (tmp_path / "exec.html").read_text() + assert "Conditional baseline (XGBoost)" not in html + assert result["conditional_comparison"] is None diff --git a/zwill/executive_summary.py b/zwill/executive_summary.py index 94ff19e..1320cea 100644 --- a/zwill/executive_summary.py +++ b/zwill/executive_summary.py @@ -319,6 +319,43 @@ def svg_header(width: int, height: int, title: str) -> list[str]: ] +def conditional_baseline_comparison( + twin_rows: list[dict[str, Any]], baseline_rows: list[dict[str, Any]] | None +) -> dict[str, Any] | None: + """Twin vs. the XGBoost conditional baseline — the deployable bar. + + Returns aggregate baseline metrics (for the Main Evidence column) plus a + paired win-rate/delta over the (respondent, question) rows both scored, or + ``None`` when no baseline is present. + """ + if not baseline_rows: + return None + baseline_metrics = { + "mean_probability_actual": float(weighted_row_mean(baseline_rows, "probability_actual") or 0.0), + "mean_negative_log_likelihood": float(weighted_row_mean(baseline_rows, "negative_log_likelihood") or 0.0), + "mean_brier": float(weighted_row_mean(baseline_rows, "brier") or 0.0), + } + baseline_p_by_key: dict[tuple[str, str], float] = {} + for row in baseline_rows: + key = (str(row.get("respondent_id")), str(row.get("heldout_question"))) + baseline_p_by_key[key] = float(row.get("probability_actual") or 0.0) + deltas: list[float] = [] + better = 0 + for row in twin_rows: + key = (str(row.get("respondent_id")), str(row.get("heldout_question"))) + if key in baseline_p_by_key: + delta = float(row.get("probability_actual") or 0.0) - baseline_p_by_key[key] + deltas.append(delta) + if delta > 0: + better += 1 + return { + "baseline_metrics": baseline_metrics, + "matched_rows": len(deltas), + "mean_p_delta": (sum(deltas) / len(deltas)) if deltas else 0.0, + "share_twin_better": (better / len(deltas)) if deltas else 0.0, + } + + def render_html( *, survey: str, @@ -332,6 +369,7 @@ def render_html( pairwise_svg: Path, pairwise: dict[str, Any], spearman_detail: dict[str, Any], + conditional: dict[str, Any] | None = None, generated_markdown: str | None = None, generation: dict[str, Any] | None = None, ) -> str: @@ -372,6 +410,26 @@ def render_html( empirical_lift_block = "" if empirical_lift_svg and empirical_lift: empirical_lift_block = f"""

Lift Versus Empirical Marginal Oracle

Mean lift versus the empirical marginal oracle is {empirical_lift['mean_lift']:.2f}x. This stricter comparison is only available because the held-out target was observed in the validation data; it is not available for future unanswered questions.

Histogram of lift over empirical marginal probability assigned to the actual answer.""" + conditional_head = "" + conditional_p_cell = "" + conditional_nll_cell = "" + conditional_brier_cell = "" + conditional_lead = "" + if conditional: + bm = conditional["baseline_metrics"] + conditional_head = "Conditional baseline (XGBoost)" + conditional_p_cell = f"{bm['mean_probability_actual']:.1%}" + conditional_nll_cell = f"{bm['mean_negative_log_likelihood']:.3f}" + conditional_brier_cell = f"{bm['mean_brier']:.3f}" + delta = float(conditional.get("mean_p_delta", 0.0)) + share = float(conditional.get("share_twin_better", 0.0)) + verdict = "beats" if delta > 0.005 else ("ties" if delta >= -0.005 else "trails") + conditional_lead = ( + f"

Versus the deployable bar: the twin {verdict} the XGBoost conditional baseline " + f"(embeddings + demographics, leave-one-question-out) — it assigns {delta:+.1%} probability to the actual " + f"answer relative to that baseline and wins on {share:.0%} of matched predictions. This is the comparison " + f"that matters; uniform and the empirical oracle are context.

" + ) per_question_rows = "".join( f"{escape(row['question'])}{row['rows']}{row['observed_mean_p_actual']:.3f}{row['null_mean_p_actual_mean']:.3f}{row['p_value_mean_p_actual']:.5f}" for row in individual.get("per_question", []) @@ -421,7 +479,7 @@ def render_html(
{metrics['row_count']:,.0f}
Validation rows
{metrics['question_count']:,.0f}
Held-out questions
{lift['share_above_1']:.0%}
Rows above uniform

Decision Guidance

Decision useRecommendationWhy
Exploratory concept screeningUse cautiouslyThe validation shows lift over uniform guessing, but {individual_signal_text.lower()}
Ranking options or messagesPreliminary directional use{ranking_why}
Exact market sizing, targeting, or public claimsDo not use aloneHeld-out validation supports aggregate/directional signal, not precise standalone estimates or individual-level action.

What Was Held Out?

The validation held out observed answers and predicted them from the remaining respondent context. Unless a run report records more specific exclusions, treat the context policy as all available observed answers except the current held-out target.

{question_rows}
QuestionHeld-out target
-

Main Evidence

MetricTwinUniform over options
Mean probability assigned to actual answer{metrics['mean_probability_actual']:.1%}{metrics['mean_uniform_probability_actual']:.1%}
Negative log likelihood{metrics['mean_negative_log_likelihood']:.3f}{metrics['mean_uniform_negative_log_likelihood']:.3f}
Brier score{metrics['mean_brier']:.3f}{metrics['mean_uniform_brier']:.3f}
+

Main Evidence

{conditional_lead}{conditional_head}{conditional_p_cell}{conditional_nll_cell}{conditional_brier_cell}
MetricTwinUniform over options
Mean probability assigned to actual answer{metrics['mean_probability_actual']:.1%}{metrics['mean_uniform_probability_actual']:.1%}
Negative log likelihood{metrics['mean_negative_log_likelihood']:.3f}{metrics['mean_uniform_negative_log_likelihood']:.3f}
Brier score{metrics['mean_brier']:.3f}{metrics['mean_uniform_brier']:.3f}

Accuracy Lift Distribution

Lift Versus Uniform

Mean lift over uniform is {lift['mean_lift']:.2f}x, median lift is {lift['median_lift']:.2f}x, and {lift['share_above_1']:.1%} of rows are above the uniform baseline. This asks whether twins beat random guessing over answer options.

Histogram of lift over uniform probability assigned to the actual answer.{empirical_lift_block}

Individual Signal Beyond Marginals

Within-question permutation keeps each prediction vector fixed and shuffles actual answers across respondents. It tests respondent-specific matching beyond question-level marginal structure; it does not test whether predictions beat uniform. A low p-value means respondent-specific matching is stronger than shuffled labels. A high p-value with good uniform lift means the model may be capturing aggregate or marginal structure rather than individual-level signal.

StatisticObservedPermutation nullp-value
Mean probability assigned to actual answer{individual['observed_mean_p_actual']:.3f}{individual['null_mean_p_actual_mean']:.3f}{individual['p_value_mean_p_actual']:.5f}
Mean negative log likelihood{individual['observed_mean_nll']:.3f}{individual['null_mean_nll_mean']:.3f}{individual['p_value_mean_nll']:.5f}

Per-question permutation results

{per_question_rows}
QuestionRowsObserved p(actual)Null p(actual)p-value

Marginal Rank Order

Plain-English readout: {ranking_readout}

Bar chart showing pairwise option ordering accuracy by validation question.

{spearman_sentence}

@@ -438,6 +496,7 @@ def build_executive_summary( markdown_path: Path | None, simulations: int, seed: int, + baseline_rows: list[dict[str, Any]] | None = None, generated_markdown: str | None = None, generation: dict[str, Any] | None = None, ) -> dict[str, Any]: @@ -497,6 +556,7 @@ def _wmean(key: str) -> float: groups = aggregate_groups(rows) pairwise = write_pairwise_order_chart(groups, pairwise_svg) spearman_detail = write_spearman_detail(groups, base.with_name(f"{prefix}_marginal_rank_order.svg"), simulations=max(100, min(simulations, 5000)), seed=seed) + conditional = conditional_baseline_comparison(rows, baseline_rows) path.write_text( render_html( survey=survey, @@ -510,6 +570,7 @@ def _wmean(key: str) -> float: pairwise_svg=pairwise_svg, pairwise=pairwise, spearman_detail=spearman_detail, + conditional=conditional, generated_markdown=generated_markdown, generation=generation, ) @@ -538,6 +599,7 @@ def _wmean(key: str) -> float: "metrics": metrics, "lift": lift, "empirical_lift": empirical_lift, + "conditional_comparison": conditional, "individual_signal": individual, "pairwise_ordering": pairwise, "spearman_rank_order": spearman_detail, diff --git a/zwill/report_bundle.py b/zwill/report_bundle.py index e018118..a92933f 100644 --- a/zwill/report_bundle.py +++ b/zwill/report_bundle.py @@ -665,6 +665,15 @@ def esc(value: Any) -> str: lift = executive.get("lift") or {} individual = executive.get("individual_signal") or {} pairwise = executive.get("pairwise_ordering") or {} + conditional = executive.get("conditional_comparison") or {} + conditional_row = "" + if conditional: + conditional_row = ( + "Beats conditional baseline" + f"{float(conditional.get('share_twin_better', 0.0)):.0%}" + "p(actual) vs conditional baseline" + f"{float(conditional.get('mean_p_delta', 0.0)):+.1%}" + ) executive_page = next((page for page in payload.get("pages", []) if page.get("page_id") == "executive-summary"), {}) executive_href = bundle_rel_link(executive_page.get("path"), output_dir) if executive_page.get("path") else "" validation_page = next((page for page in payload.get("pages", []) if page.get("page_id") == "twin-validation"), {}) @@ -678,6 +687,7 @@ def esc(value: Any) -> str: Validation rows{esc(int(metrics.get("row_count", 0)))}Held-out questions{esc(int(metrics.get("question_count", 0)))} Mean p(actual){float(metrics.get("mean_probability_actual", 0.0)):.1%}Uniform p(actual){float(metrics.get("mean_uniform_probability_actual", 0.0)):.1%} + {conditional_row} Rows above uniform{float(lift.get("share_above_1", 0.0)):.0%}Mean lift vs uniform{float(lift.get("mean_lift", 0.0)):.2f}x Individual-signal p-value{float(individual.get("p_value_mean_p_actual", 0.0)):.5f}Option-pair ordering accuracy{float((pairwise.get("summary") or {}).get("pairwise_order_accuracy", 0.0)):.0%} @@ -993,6 +1003,7 @@ def add_page(page: dict[str, Any]) -> None: markdown_path=executive_markdown_path, simulations=getattr(args, "permutations", DEFAULT_REPORT_PERMUTATIONS), seed=getattr(args, "seed", 20260701), + baseline_rows=baseline_rows, generated_markdown=generated_executive.get("markdown") if generated_executive else None, generation=generated_executive.get("generation") if generated_executive else None, )