diff --git a/tests/test_report_conditional_baseline.py b/tests/test_report_conditional_baseline.py
index 8032ef1..7a66c47 100644
--- a/tests/test_report_conditional_baseline.py
+++ b/tests/test_report_conditional_baseline.py
@@ -3,12 +3,14 @@
import math
from zwill.cli import build_twin_report
+from zwill.executive_summary import build_executive_summary
from zwill.reporting import render_twin_summary_report_html
from zwill.twin_baseline import MODEL_LABEL as BASELINE_MODEL_LABEL
-def _row(model_label: str, respondent: str, actual: str, nll: float, correct: int) -> dict:
- probs = {"A": 0.8, "B": 0.2} if actual == "A" else {"A": 0.2, "B": 0.8}
+def _row(model_label: str, respondent: str, actual: str, nll: float, correct: int, p_actual: float = 0.8) -> dict:
+ other = "B" if actual == "A" else "A"
+ probs = {actual: p_actual, other: 1.0 - p_actual}
return {
"job_id": "twin1" if model_label != BASELINE_MODEL_LABEL else "base1",
"survey": "demo",
@@ -22,7 +24,7 @@ def _row(model_label: str, respondent: str, actual: str, nll: float, correct: in
"option_labels": ["A", "B"],
"probabilities": probs,
"raw_probabilities": [probs["A"], probs["B"]],
- "probability_actual": probs[actual],
+ "probability_actual": p_actual,
"uniform_probability_actual": 0.5,
"uniform_negative_log_likelihood": math.log(2),
"negative_log_likelihood": nll,
@@ -67,3 +69,36 @@ def test_no_conditional_column_when_baseline_absent() -> None:
html = _render(twin)
assert "NLL improvement vs conditional baseline" not in html
assert "Conditional baseline (XGBoost)" not in html
+
+
+def test_executive_summary_features_conditional_baseline(tmp_path) -> None:
+ twin = [_row("openai:gpt-5.5", "r1", "A", 0.20, 1, p_actual=0.8), _row("openai:gpt-5.5", "r2", "B", 0.25, 1, p_actual=0.8)]
+ baseline = [_row(BASELINE_MODEL_LABEL, "r1", "A", 0.55, 1, p_actual=0.5), _row(BASELINE_MODEL_LABEL, "r2", "B", 0.60, 0, p_actual=0.5)]
+ result = build_executive_summary(
+ twin,
+ survey="demo",
+ path=tmp_path / "exec.html",
+ markdown_path=None,
+ simulations=25,
+ seed=1,
+ baseline_rows=baseline,
+ )
+ html = (tmp_path / "exec.html").read_text()
+ # Main Evidence gains the conditional baseline as a column + a lead sentence.
+ assert "Conditional baseline (XGBoost)" in html
+ assert "Versus the deployable bar" in html
+ # The comparison is exposed for the index tile.
+ comp = result["conditional_comparison"]
+ assert comp is not None and comp["matched_rows"] == 2
+ # Twin (p_actual 0.8/0.8) clearly beats baseline here.
+ assert comp["share_twin_better"] == 1.0
+
+
+def test_executive_summary_unchanged_without_baseline(tmp_path) -> None:
+ twin = [_row("openai:gpt-5.5", "r1", "A", 0.20, 1), _row("openai:gpt-5.5", "r2", "B", 0.25, 1)]
+ result = build_executive_summary(
+ twin, survey="demo", path=tmp_path / "exec.html", markdown_path=None, simulations=25, seed=1
+ )
+ html = (tmp_path / "exec.html").read_text()
+ assert "Conditional baseline (XGBoost)" not in html
+ assert result["conditional_comparison"] is None
diff --git a/zwill/executive_summary.py b/zwill/executive_summary.py
index 94ff19e..1320cea 100644
--- a/zwill/executive_summary.py
+++ b/zwill/executive_summary.py
@@ -319,6 +319,43 @@ def svg_header(width: int, height: int, title: str) -> list[str]:
]
+def conditional_baseline_comparison(
+ twin_rows: list[dict[str, Any]], baseline_rows: list[dict[str, Any]] | None
+) -> dict[str, Any] | None:
+ """Twin vs. the XGBoost conditional baseline — the deployable bar.
+
+ Returns aggregate baseline metrics (for the Main Evidence column) plus a
+ paired win-rate/delta over the (respondent, question) rows both scored, or
+ ``None`` when no baseline is present.
+ """
+ if not baseline_rows:
+ return None
+ baseline_metrics = {
+ "mean_probability_actual": float(weighted_row_mean(baseline_rows, "probability_actual") or 0.0),
+ "mean_negative_log_likelihood": float(weighted_row_mean(baseline_rows, "negative_log_likelihood") or 0.0),
+ "mean_brier": float(weighted_row_mean(baseline_rows, "brier") or 0.0),
+ }
+ baseline_p_by_key: dict[tuple[str, str], float] = {}
+ for row in baseline_rows:
+ key = (str(row.get("respondent_id")), str(row.get("heldout_question")))
+ baseline_p_by_key[key] = float(row.get("probability_actual") or 0.0)
+ deltas: list[float] = []
+ better = 0
+ for row in twin_rows:
+ key = (str(row.get("respondent_id")), str(row.get("heldout_question")))
+ if key in baseline_p_by_key:
+ delta = float(row.get("probability_actual") or 0.0) - baseline_p_by_key[key]
+ deltas.append(delta)
+ if delta > 0:
+ better += 1
+ return {
+ "baseline_metrics": baseline_metrics,
+ "matched_rows": len(deltas),
+ "mean_p_delta": (sum(deltas) / len(deltas)) if deltas else 0.0,
+ "share_twin_better": (better / len(deltas)) if deltas else 0.0,
+ }
+
+
def render_html(
*,
survey: str,
@@ -332,6 +369,7 @@ def render_html(
pairwise_svg: Path,
pairwise: dict[str, Any],
spearman_detail: dict[str, Any],
+ conditional: dict[str, Any] | None = None,
generated_markdown: str | None = None,
generation: dict[str, Any] | None = None,
) -> str:
@@ -372,6 +410,26 @@ def render_html(
empirical_lift_block = ""
if empirical_lift_svg and empirical_lift:
empirical_lift_block = f"""Lift Versus Empirical Marginal Oracle
Mean lift versus the empirical marginal oracle is {empirical_lift['mean_lift']:.2f}x. This stricter comparison is only available because the held-out target was observed in the validation data; it is not available for future unanswered questions.
})
"""
+ conditional_head = ""
+ conditional_p_cell = ""
+ conditional_nll_cell = ""
+ conditional_brier_cell = ""
+ conditional_lead = ""
+ if conditional:
+ bm = conditional["baseline_metrics"]
+ conditional_head = "Conditional baseline (XGBoost) | "
+ conditional_p_cell = f"{bm['mean_probability_actual']:.1%} | "
+ conditional_nll_cell = f"{bm['mean_negative_log_likelihood']:.3f} | "
+ conditional_brier_cell = f"{bm['mean_brier']:.3f} | "
+ delta = float(conditional.get("mean_p_delta", 0.0))
+ share = float(conditional.get("share_twin_better", 0.0))
+ verdict = "beats" if delta > 0.005 else ("ties" if delta >= -0.005 else "trails")
+ conditional_lead = (
+ f"Versus the deployable bar: the twin {verdict} the XGBoost conditional baseline "
+ f"(embeddings + demographics, leave-one-question-out) — it assigns {delta:+.1%} probability to the actual "
+ f"answer relative to that baseline and wins on {share:.0%} of matched predictions. This is the comparison "
+ f"that matters; uniform and the empirical oracle are context.
"
+ )
per_question_rows = "".join(
f"{escape(row['question'])} | {row['rows']} | {row['observed_mean_p_actual']:.3f} | {row['null_mean_p_actual_mean']:.3f} | {row['p_value_mean_p_actual']:.5f} |
"
for row in individual.get("per_question", [])
@@ -421,7 +479,7 @@ def render_html(
{metrics['row_count']:,.0f}
Validation rows
{metrics['question_count']:,.0f}
Held-out questions
{lift['share_above_1']:.0%}
Rows above uniform
Decision Guidance
| Decision use | Recommendation | Why |
|---|
| Exploratory concept screening | Use cautiously | The validation shows lift over uniform guessing, but {individual_signal_text.lower()} |
| Ranking options or messages | Preliminary directional use | {ranking_why} |
| Exact market sizing, targeting, or public claims | Do not use alone | Held-out validation supports aggregate/directional signal, not precise standalone estimates or individual-level action. |
What Was Held Out?
The validation held out observed answers and predicted them from the remaining respondent context. Unless a run report records more specific exclusions, treat the context policy as all available observed answers except the current held-out target.
| Question | Held-out target |
|---|
{question_rows}
Main Evidence
| Metric | Twin | Uniform over options |
|---|
| Mean probability assigned to actual answer | {metrics['mean_probability_actual']:.1%} | {metrics['mean_uniform_probability_actual']:.1%} |
| Negative log likelihood | {metrics['mean_negative_log_likelihood']:.3f} | {metrics['mean_uniform_negative_log_likelihood']:.3f} |
| Brier score | {metrics['mean_brier']:.3f} | {metrics['mean_uniform_brier']:.3f} |
Main Evidence
{conditional_lead}
| Metric | Twin | {conditional_head}Uniform over options |
|---|
| Mean probability assigned to actual answer | {metrics['mean_probability_actual']:.1%} | {conditional_p_cell}{metrics['mean_uniform_probability_actual']:.1%} |
| Negative log likelihood | {metrics['mean_negative_log_likelihood']:.3f} | {conditional_nll_cell}{metrics['mean_uniform_negative_log_likelihood']:.3f} |
| Brier score | {metrics['mean_brier']:.3f} | {conditional_brier_cell}{metrics['mean_uniform_brier']:.3f} |
Accuracy Lift Distribution
Lift Versus Uniform
Mean lift over uniform is {lift['mean_lift']:.2f}x, median lift is {lift['median_lift']:.2f}x, and {lift['share_above_1']:.1%} of rows are above the uniform baseline. This asks whether twins beat random guessing over answer options.
})
{empirical_lift_block}
Individual Signal Beyond Marginals
Within-question permutation keeps each prediction vector fixed and shuffles actual answers across respondents. It tests respondent-specific matching beyond question-level marginal structure; it does not test whether predictions beat uniform. A low p-value means respondent-specific matching is stronger than shuffled labels. A high p-value with good uniform lift means the model may be capturing aggregate or marginal structure rather than individual-level signal.
| Statistic | Observed | Permutation null | p-value |
|---|
| Mean probability assigned to actual answer | {individual['observed_mean_p_actual']:.3f} | {individual['null_mean_p_actual_mean']:.3f} | {individual['p_value_mean_p_actual']:.5f} |
| Mean negative log likelihood | {individual['observed_mean_nll']:.3f} | {individual['null_mean_nll_mean']:.3f} | {individual['p_value_mean_nll']:.5f} |
Per-question permutation results
| Question | Rows | Observed p(actual) | Null p(actual) | p-value |
|---|
{per_question_rows}
Marginal Rank Order
Plain-English readout: {ranking_readout}
})
{spearman_sentence}
| Beats conditional baseline | "
+ f"{float(conditional.get('share_twin_better', 0.0)):.0%} | "
+ "p(actual) vs conditional baseline | "
+ f"{float(conditional.get('mean_p_delta', 0.0)):+.1%} |
|---|
"
+ )
executive_page = next((page for page in payload.get("pages", []) if page.get("page_id") == "executive-summary"), {})
executive_href = bundle_rel_link(executive_page.get("path"), output_dir) if executive_page.get("path") else ""
validation_page = next((page for page in payload.get("pages", []) if page.get("page_id") == "twin-validation"), {})
@@ -678,6 +687,7 @@ def esc(value: Any) -> str:
| Validation rows | {esc(int(metrics.get("row_count", 0)))} | Held-out questions | {esc(int(metrics.get("question_count", 0)))} |
|---|
| Mean p(actual) | {float(metrics.get("mean_probability_actual", 0.0)):.1%} | Uniform p(actual) | {float(metrics.get("mean_uniform_probability_actual", 0.0)):.1%} |
|---|
+ {conditional_row}
| Rows above uniform | {float(lift.get("share_above_1", 0.0)):.0%} | Mean lift vs uniform | {float(lift.get("mean_lift", 0.0)):.2f}x |
|---|
| Individual-signal p-value | {float(individual.get("p_value_mean_p_actual", 0.0)):.5f} | Option-pair ordering accuracy | {float((pairwise.get("summary") or {}).get("pairwise_order_accuracy", 0.0)):.0%} |
|---|
@@ -993,6 +1003,7 @@ def add_page(page: dict[str, Any]) -> None:
markdown_path=executive_markdown_path,
simulations=getattr(args, "permutations", DEFAULT_REPORT_PERMUTATIONS),
seed=getattr(args, "seed", 20260701),
+ baseline_rows=baseline_rows,
generated_markdown=generated_executive.get("markdown") if generated_executive else None,
generation=generated_executive.get("generation") if generated_executive else None,
)