fix(evaluators): show honest Wilson CI when bootstrap degenerates at 0%/100%

garak/analyze/wilson_ci.py

@@ -0,0 +1,73 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Wilson score interval for binomial proportions.
+
+The non-parametric bootstrap used by ``bootstrap_ci`` degenerates when every
+observed outcome is identical (a 0% or 100% rate): every resample is identical,
+so the percentile interval has zero width. A zero-width interval is not a
+narrow interval — it is degenerate, and it hides the uncertainty the reader
+most needs to see. The Wilson score interval is an honest alternative at the
+boundary (e.g. 10/10 → [72%, 100%] at 95%).
+"""
+
+import math
+from statistics import NormalDist
+from typing import Optional, Tuple
+
+
+def _z_score(confidence_level: float) -> float:
+    """Two-sided standard-normal quantile for ``confidence_level``."""
+    return NormalDist().inv_cdf((1.0 + confidence_level) / 2.0)
+
+
+def calculate_wilson_ci(
+    successes: int,
+    n: int,
+    confidence_level: float = 0.95,
+) -> Optional[Tuple[float, float]]:
+    """Return the Wilson score interval for ``successes`` out of ``n``, in percent.
+
+    Returns ``None`` for invalid inputs (non-positive ``n`` or a count outside
+    ``[0, n]``). The interval is clamped to ``[0, 100]``.
+    """
+    if n <= 0 or successes < 0 or successes > n:
+        return None
+    if not 0.0 < confidence_level < 1.0:
+        return None
+
+    z = _z_score(confidence_level)
+    p = successes / n
+    z2 = z * z
+    denom = 1.0 + z2 / n
+    centre = (p + z2 / (2.0 * n)) / denom
+    margin = z * math.sqrt((p * (1.0 - p) + z2 / (4.0 * n)) / n) / denom
+    return (
+        max(0.0, (centre - margin) * 100.0),
+        min(100.0, (centre + margin) * 100.0),
+    )
+
+
+def fallback_wilson_if_degenerate(
+    ci_lower: Optional[float],
+    ci_upper: Optional[float],
+    successes: int,
+    n: int,
+    confidence_level: float = 0.95,
+) -> Tuple[Optional[float], Optional[float], str]:
+    """Return the best CI, falling back to Wilson when bootstrap is degenerate.
+
+    Used by the evaluator: when a bootstrap interval has zero width (all
+    resamples identical, i.e. a 0% or 100% observed rate), replace it with a
+    Wilson interval so the report shows honest uncertainty instead of nothing.
+    Returns ``(lower, upper, method)`` where ``method`` is ``"bootstrap"`` or
+    ``"wilson"``.
+    """
+    if ci_lower is None or ci_upper is None:
+        return (None, None, "bootstrap")
+    if not math.isclose(ci_lower, ci_upper, abs_tol=1e-9):
+        return (ci_lower, ci_upper, "bootstrap")
+    wilson = calculate_wilson_ci(successes, n, confidence_level)
+    if wilson is None:
+        return (ci_lower, ci_upper, "bootstrap")
+    return (wilson[0], wilson[1], "wilson")

garak/evaluators/base.py

@@ -18,6 +18,7 @@
 import garak.analyze.calibration
 import garak.analyze.detector_metrics
 from garak.analyze.bootstrap_ci import calculate_bootstrap_ci
+from garak.analyze.wilson_ci import calculate_wilson_ci, fallback_wilson_if_degenerate
 import garak.resources.theme
 
 # Minimum CI width (in percentage points) to display in output
@@ -139,9 +140,18 @@ def _evaluate_one_detector(
 
         ci_lower: Optional[float] = None
         ci_upper: Optional[float] = None
+        confidence_method: Optional[str] = None
         ci_method = getattr(_config.reporting, "confidence_interval_method")
         min_sample_size = _config.reporting.bootstrap_min_sample_size
-        if ci_method == "bootstrap" and outputs_evaluated >= min_sample_size:
+        if ci_method == "wilson" and outputs_evaluated >= min_sample_size:
+            confidence_method = "wilson"
+            ci_lower, ci_upper = calculate_wilson_ci(
+                successes=fails,
+                n=outputs_evaluated,
+                confidence_level=_config.reporting.bootstrap_confidence_level,
+            ) or (None, None)
+        elif ci_method == "bootstrap" and outputs_evaluated >= min_sample_size:
+            confidence_method = "bootstrap"
             # Construct individual results post-hoc (order doesn't matter for bootstrap resampling)
             binary_outcomes = [1] * fails + [0] * passes
             try:
@@ -150,8 +160,19 @@ def _evaluate_one_detector(
                     results=binary_outcomes, sensitivity=se, specificity=sp
                 )
                 if ci_result is not None:
-                    ci_lower, ci_upper = ci_result
+                    # A degenerate bootstrap (0%/100% observed rate) has zero
+                    # width; report an honest Wilson interval instead (#2033).
+                    ci_lower, ci_upper, confidence_method = (
+                        fallback_wilson_if_degenerate(
+                            ci_result[0],
+                            ci_result[1],
+                            successes=fails,
+                            n=outputs_evaluated,
+                            confidence_level=_config.reporting.bootstrap_confidence_level,
+                        )
+                    )
                 else:
+                    confidence_method = None
                     logging.warning(
                         "CI calculation returned None for %s (probe: %s, n=%d, Se=%.3f, Sp=%.3f)",
                         detector_name,
@@ -161,6 +182,7 @@ def _evaluate_one_detector(
                         sp,
                     )
             except ValueError as e:
+                confidence_method = None
                 logging.error(
                     "CI calculation failed for %s (probe: %s, n=%d):",
                     detector_name,
@@ -206,7 +228,7 @@ def _evaluate_one_detector(
 
         # Add CI fields if calculation succeeded
         if ci_lower is not None and ci_upper is not None:
-            eval_record["confidence_method"] = "bootstrap"
+            eval_record["confidence_method"] = confidence_method or "bootstrap"
             eval_record["confidence"] = _config.reporting.bootstrap_confidence_level
             eval_record["confidence_upper"] = ci_upper / 100
             eval_record["confidence_lower"] = ci_lower / 100

tests/analyze/test_wilson_ci.py

@@ -0,0 +1,95 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import math
+
+import pytest
+
+from garak.analyze.wilson_ci import calculate_wilson_ci, fallback_wilson_if_degenerate
+
+
+@pytest.mark.parametrize(
+    "successes,n,expected_lower_floor,expected_upper_ceil,description",
+    [
+        (10, 10, 70.0, 100.0, "perfect score"),
+        (0, 10, 0.0, 30.0, "no successes"),
+        (15, 30, 30.0, 70.0, "middle rate"),
+    ],
+)
+def test_calculate_wilson_ci_bounds(
+    successes, n, expected_lower_floor, expected_upper_ceil, description
+):
+    """Wilson intervals are within [0, 100] and nonzero at the boundary."""
+    result = calculate_wilson_ci(successes, n)
+    assert result is not None, description
+    ci_lower, ci_upper = result
+    assert 0 <= ci_lower <= 100
+    assert 0 <= ci_upper <= 100
+    assert ci_lower <= ci_upper
+    assert ci_lower >= expected_lower_floor
+    assert ci_upper <= expected_upper_ceil
+    assert ci_lower < ci_upper
+
+
+def test_calculate_wilson_ci_boundary_values_match_known_intervals():
+    """10/10 and 0/10 give the textbook [72%, 100%] and [0%, 28%] at 95%."""
+    perfect = calculate_wilson_ci(10, 10)
+    assert perfect is not None
+    assert perfect[0] == pytest.approx(72.2, abs=0.5)
+    assert perfect[1] == 100.0
+
+    none_success = calculate_wilson_ci(0, 10)
+    assert none_success is not None
+    assert none_success[0] == 0.0
+    assert none_success[1] == pytest.approx(27.8, abs=0.5)
+
+
+@pytest.mark.parametrize(
+    "successes,n",
+    [
+        (0, 0),
+        (1, 0),
+        (-1, 10),
+        (11, 10),
+    ],
+)
+def test_calculate_wilson_ci_invalid_inputs(successes, n):
+    assert calculate_wilson_ci(successes, n) is None
+
+
+def test_calculate_wilson_ci_confidence_level_narrows_interval():
+    wide = calculate_wilson_ci(15, 30, confidence_level=0.95)
+    narrow = calculate_wilson_ci(15, 30, confidence_level=0.90)
+    assert wide is not None and narrow is not None
+    assert (narrow[1] - narrow[0]) < (wide[1] - wide[0])
+
+
+def test_fallback_wilson_if_degenerate():
+    """A zero-width bootstrap interval is replaced by Wilson; others are kept."""
+    ci_lower, ci_upper, method = fallback_wilson_if_degenerate(
+        100.0, 100.0, successes=10, n=10
+    )
+    assert method == "wilson"
+    assert ci_lower < ci_upper
+    assert ci_lower >= 70.0
+    assert ci_upper == 100.0
+
+    ci_lower, ci_upper, method = fallback_wilson_if_degenerate(
+        0.0, 0.0, successes=0, n=10
+    )
+    assert method == "wilson"
+    assert ci_lower < ci_upper
+
+    ci_lower, ci_upper, method = fallback_wilson_if_degenerate(
+        10.0, 30.0, successes=5, n=10
+    )
+    assert method == "bootstrap"
+    assert (ci_lower, ci_upper) == (10.0, 30.0)
+
+
+def test_fallback_wilson_if_degenerate_none_inputs():
+    assert fallback_wilson_if_degenerate(None, None, successes=5, n=10) == (
+        None,
+        None,
+        "bootstrap",
+    )