diff --git a/docs/reference/quantile_ci.md b/docs/reference/quantile_ci.md new file mode 100644 index 0000000..b7214b8 --- /dev/null +++ b/docs/reference/quantile_ci.md @@ -0,0 +1,9 @@ +# llmeter.quantile_ci + +::: llmeter.quantile_ci + options: + show_root_heading: true + show_source: true + members: + - quantile_ci + - can_estimate_quantile diff --git a/docs/user_guide/quantile_confidence.md b/docs/user_guide/quantile_confidence.md new file mode 100644 index 0000000..caefc02 --- /dev/null +++ b/docs/user_guide/quantile_confidence.md @@ -0,0 +1,109 @@ +# Quantile Confidence Intervals + +## Motivation + +When you report a percentile like p99 from a test run, the **reliability** of that +estimate depends on how many data points you collected. With only 50 samples, your +p99 value is based on a single observation — any outlier (network glitch, cold start, +garbage collection pause) dominates the estimate. + +LLMeter can optionally compute confidence intervals for quantile estimates using the +exact binomial method. This feature also acts as a **reliability gate**: when the sample +size is too small to form a meaningful confidence interval, the quantile is omitted from +the output rather than reported with false precision. + +## Enabling confidence intervals + +Pass `confidence` to `summary_stats_from_list`, or set it on the `Runner`: + +```python +from llmeter.utils import summary_stats_from_list + +data = [resp.time_to_first_token for resp in result.responses if resp.time_to_first_token] + +# With 95% confidence interval estimation and gating +stats = summary_stats_from_list(data, percentiles=(50, 90, 99), confidence=0.95) +``` + +When `confidence` is set: + +1. **Gating** — quantiles that cannot achieve the requested confidence level are + silently omitted from the output. For example, with `confidence=0.95`, p99 requires + at least ~473 samples. If you only have 100, `p99` won't appear in the dict. + +2. **CI bounds** — for quantiles that pass the gate, lower and upper bounds are + included: + ```python + stats["p90"] # 0.612 (point estimate) + stats["p90_ci_lower"] # 0.581 (lower bound of 95% CI) + stats["p90_ci_upper"] # 0.647 (upper bound of 95% CI) + ``` + +Without `confidence` (the default), all requested percentiles are always reported +with no CI bounds — the legacy behavior is unchanged. + +## Minimum sample sizes + +The minimum number of samples required to form a confidence interval at 95% confidence: + +| Quantile | Min samples | Practical guidance | +|----------|-------------|-------------------| +| p50 | 8 | Almost always available | +| p90 | 46 | Small test runs may miss this | +| p95 | 90 | Need ~100 successful requests | +| p99 | 482 | Need ~500 successful requests | + +These are derived from the exact binomial distribution — they represent the smallest +sample sizes where the CI achieves ≥95% actual coverage probability. + +!!! tip "Sizing your test runs" + If p99 latency matters for your SLOs and you want confidence intervals, + aim for at least 500 successful requests per run. For p90, 50 requests is sufficient. + +## How it works + +The method is based on the relationship between order statistics and the binomial +distribution: + +1. Sort the n observations: X₍₁₎ ≤ X₍₂₎ ≤ ... ≤ X₍ₙ₎ +2. The number of observations below the true p-quantile follows Binomial(n, p) +3. Find indices `l` and `u` such that P(l ≤ Binomial(n,p) ≤ u-1) ≥ confidence +4. The CI is [X₍ₗ₊₁₎, X₍ᵤ₊₁₎] + +The implementation uses log-space arithmetic (`math.lgamma`) to avoid integer overflow +for large n, with zero external dependencies beyond the Python standard library. + +!!! note "Coverage vs. nominal confidence" + Because the binomial distribution is discrete, the actual coverage of the CI + may slightly exceed the nominal confidence level. For example, the first n where + a 95% CI is achievable for p50 is n=8, which gives 96.1% actual coverage. The + implementation verifies that actual coverage meets the threshold rather than relying + on a naive index check. + +## Example: interpreting gated output + +```python +result = await runner.run(payload=payload, n_requests=30, clients=5) + +# Without confidence (default) — all percentiles reported +stats = summary_stats_from_list( + [r.time_to_first_token for r in result.responses if r.time_to_first_token], + percentiles=(50, 90, 99), +) +# → {"average": 0.42, "p50": 0.38, "p90": 0.61, "p99": 1.23} + +# With confidence=0.95 — unreliable percentiles gated out +stats = summary_stats_from_list( + [r.time_to_first_token for r in result.responses if r.time_to_first_token], + percentiles=(50, 90, 99), + confidence=0.95, +) +# With only 30 samples: +# → {"average": 0.42, "p50": 0.38, "p50_ci_lower": 0.31, "p50_ci_upper": 0.44} +# p90 and p99 are omitted because n=30 is insufficient +``` + +## API reference + +::: llmeter.quantile_ci.quantile_ci +::: llmeter.quantile_ci.can_estimate_quantile diff --git a/llmeter/quantile_ci.py b/llmeter/quantile_ci.py new file mode 100644 index 0000000..f3bea1b --- /dev/null +++ b/llmeter/quantile_ci.py @@ -0,0 +1,192 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Quantile confidence interval estimation — zero external dependencies. + +Provides functions to compute confidence intervals for quantiles using the +exact binomial method, and to gate unreliable quantile estimates based on +sample size. Only uses the Python standard library (math.lgamma). + +Reference: The confidence interval for a quantile is based on order statistics +of the binomial distribution. If X_(1), ..., X_(n) are sorted i.i.d. samples, +then [X_(l+1), X_(u+1)] covers the true p-quantile with probability: + + P(l <= Binomial(n, p) <= u - 1) + +See: https://en.wikipedia.org/wiki/Quantile#Confidence_intervals +""" + +from math import exp, lgamma, log +from typing import Optional, Sequence, Tuple + + +def _log_binom_pmf(k: int, n: int, p: float) -> float: + """Log of the binomial PMF: log(C(n,k) * p^k * (1-p)^(n-k)). + + Uses lgamma to avoid integer overflow for large n. + """ + if p == 0: + return 0.0 if k == 0 else float("-inf") + if p == 1: + return 0.0 if k == n else float("-inf") + return ( + lgamma(n + 1) + - lgamma(k + 1) + - lgamma(n - k + 1) + + k * log(p) + + (n - k) * log(1 - p) + ) + + +def _binom_cdf(k: int, n: int, p: float) -> float: + """Cumulative distribution function P(X <= k) for Binomial(n, p). + + Uses log-space arithmetic per term to avoid overflow. + """ + if k < 0: + return 0.0 + if k >= n: + return 1.0 + cumulative = 0.0 + for i in range(k + 1): + cumulative += exp(_log_binom_pmf(i, n, p)) + return cumulative + + +def _binom_ppf(q: float, n: int, p: float) -> int: + """Percent point function (inverse CDF) for Binomial(n, p). + + Returns the smallest k such that P(X <= k) >= q. + Complexity: O(n) — negligible for n < 10,000. + """ + if q <= 0: + return 0 + if q >= 1: + return n + + cumulative = 0.0 + for k in range(n + 1): + cumulative += exp(_log_binom_pmf(k, n, p)) + if cumulative >= q: + return k + return n + + +def _ci_coverage(lower_idx: int, upper_idx: int, n: int, p: float) -> float: + """Compute actual coverage probability of a quantile CI. + + The interval [X_(lower+1), X_(upper+1)] covers the true p-quantile + when the number of samples below it (Binomial(n, p)) falls in + [lower_idx, upper_idx - 1]. + + Coverage = P(lower_idx <= X <= upper_idx - 1) + = CDF(upper_idx - 1) - CDF(lower_idx - 1) + """ + cdf_upper = _binom_cdf(upper_idx - 1, n, p) + cdf_lower = _binom_cdf(lower_idx - 1, n, p) if lower_idx > 0 else 0.0 + return cdf_upper - cdf_lower + + +def _find_ci_indices( + n: int, quantile: float, confidence: float +) -> Optional[Tuple[int, int]]: + """Find order-statistic indices that achieve the requested coverage. + + Uses the binomial ppf as a starting point, then verifies actual coverage. + Returns (lower_idx, upper_idx) where the CI is + [sorted_data[lower_idx], sorted_data[upper_idx]], or None if no valid + interval exists within the data range. + """ + alpha = 1 - confidence + + # Starting point from binomial inverse CDF: + # lower_idx: one below the alpha/2 quantile of Binom(n, p) + # upper_idx: the (1 - alpha/2) quantile of Binom(n, p) + lower_idx = max(0, _binom_ppf(alpha / 2, n, quantile) - 1) + upper_idx = min(n - 1, _binom_ppf(1 - alpha / 2, n, quantile)) + + if lower_idx >= upper_idx: + return None + + # Verify actual coverage meets the confidence threshold + coverage = _ci_coverage(lower_idx, upper_idx, n, quantile) + if coverage >= confidence: + return (lower_idx, upper_idx) + + # If coverage is insufficient, try widening by one on each side + # (accounts for discreteness of the binomial distribution) + candidates = [] + if lower_idx > 0: + c = _ci_coverage(lower_idx - 1, upper_idx, n, quantile) + if c >= confidence: + candidates.append((lower_idx - 1, upper_idx, c)) + if upper_idx < n - 1: + c = _ci_coverage(lower_idx, upper_idx + 1, n, quantile) + if c >= confidence: + candidates.append((lower_idx, upper_idx + 1, c)) + + if not candidates: + return None + + # Pick the tightest interval (smallest coverage above threshold) + best = min(candidates, key=lambda x: x[2]) + return (best[0], best[1]) + + +def quantile_ci( + data: Sequence[float], + quantile: float, + confidence: float = 0.95, +) -> Optional[Tuple[float, float]]: + """Compute a confidence interval for the given quantile of a dataset. + + Uses the exact binomial method to find order statistic indices that + bracket the true quantile. Verifies that the actual coverage probability + meets the requested confidence level (accounting for the discreteness + of the binomial distribution). + + Args: + data: Sequence of observed values (e.g., latencies in seconds). + quantile: The quantile of interest (e.g., 0.9 for p90, 0.99 for p99). + confidence: Desired confidence level (default 0.95 for 95% CI). + + Returns: + A tuple (lower_bound, upper_bound) representing the confidence interval, + or None if the sample size is too small to achieve the requested coverage. + """ + sorted_data = sorted(data) + n = len(sorted_data) + + if n < 2: + return None + + result = _find_ci_indices(n, quantile, confidence) + if result is None: + return None + + lower_idx, upper_idx = result + return (sorted_data[lower_idx], sorted_data[upper_idx]) + + +def can_estimate_quantile( + n: int, + quantile: float, + confidence: float = 0.95, +) -> bool: + """Check whether a CI can be formed for the given quantile and sample size. + + Verifies that the actual achievable coverage meets the requested confidence + level. Useful as a validity gate: if this returns False, the point estimate + of the quantile should be considered unreliable and may be omitted/flagged. + + Args: + n: Sample size. + quantile: The quantile of interest (e.g. 0.99 for p99). + confidence: Desired confidence level. + + Returns: + True if a confidence interval with adequate coverage can be formed. + """ + if n < 2: + return False + return _find_ci_indices(n, quantile, confidence) is not None diff --git a/llmeter/utils.py b/llmeter/utils.py index 12f5efd..22f44e5 100644 --- a/llmeter/utils.py +++ b/llmeter/utils.py @@ -11,6 +11,9 @@ from upath.types import ReadablePathLike, WritablePathLike +from .quantile_ci import can_estimate_quantile, quantile_ci + + class DeferredError: """Stores an exception and raises it at a later time if this object is accessed in any way. @@ -28,12 +31,6 @@ class DeferredError: """ def __init__(self, exception): - # # Ensure the exception is a BaseException instance - # if isinstance(exception, BaseException): - # self.exc = exception - # else: - # # If it's not a BaseException, wrap it in an ImportError - # self.exc = ImportError(str(exception)) self.exc = exception def __getattr__(self, name: str): @@ -50,24 +47,38 @@ def __getattr__(self, name: str): def summary_stats_from_list( data: Sequence[int | float], percentiles: Sequence[int] = (50, 90, 99), + confidence: float | None = None, ) -> dict[str, int | float]: """Calculate summary statistics for a sequence of numbers in pure Python Args: data: Sequence of numbers percentiles: Integer percentiles from 1-99 + confidence: Confidence level for quantile CI estimation (default 0.95). + When set (e.g. ``confidence=0.95``), quantiles whose confidence + interval cannot be formed at this level are omitted from the output, + and CI bounds are attached as ``p{N}_ci_lower`` / ``p{N}_ci_upper`` + keys. Default is None (legacy behavior: always report point estimates). Returns: stats: Dictionary of descriptive statistics including "average" (the arithmetic mean of the input `data`), and the requested `percentiles` in format "p50", "p90", "p99", etc. + When confidence gating is enabled, unreliable quantiles are omitted. """ clean_data = list(filterfalse(isnan, data)) + n = len(clean_data) try: result = dict(average=mean(clean_data)) # Rather than always using n=100-based quantiles, we'll adapt between a few specific bases # depending on the requested percentiles - and calculate these splits only as needed: q_bases: dict[int, Any] = {k: None for k in [4, 10, 100]} for p in percentiles: - if len(clean_data) == 1: + q = p / 100 + + # Gate: skip quantiles we can't reliably estimate + if confidence is not None and not can_estimate_quantile(n, q, confidence): + continue + + if n == 1: result[f"p{p}"] = clean_data[0] elif p == 50: result[f"p{p}"] = median(clean_data) @@ -81,6 +92,14 @@ def summary_stats_from_list( if q_bases[q_base] is None: q_bases[q_base] = quantiles(clean_data, n=q_base) result[f"p{p}"] = q_bases[q_base][int(p * q_base / 100) - 1] + + # Attach CI bounds when available + if confidence is not None and n > 1: + ci = quantile_ci(clean_data, q, confidence) + if ci: + result[f"p{p}_ci_lower"] = ci[0] + result[f"p{p}_ci_upper"] = ci[1] + return result except StatisticsError: @@ -225,6 +244,14 @@ def _send_window(self) -> float | None: return (self._last_send_time - self._first_send_time).total_seconds() return None + def snapshot(self) -> dict[str, Any]: + """Return current stats for live display without finalizing. + + This is called repeatedly during a run to update the progress bar. + Does not require ``end_time`` or ``result_dict``. + """ + return self.to_stats() + def now_utc() -> datetime: """Returns the current UTC datetime. @@ -236,27 +263,17 @@ def now_utc() -> datetime: @overload -def ensure_path(path: ReadablePathLike | WritablePathLike) -> UPath: ... +def ensure_path(path: None) -> None: ... @overload -def ensure_path(path: None) -> None: ... +def ensure_path(path: ReadablePathLike | WritablePathLike) -> UPath: ... def ensure_path( path: ReadablePathLike | WritablePathLike | None, ) -> UPath | None: - """Normalize a path-like argument to a UPath instance. - - Converts strings, os.PathLike objects, and UPath instances into a - consistent UPath representation. Passes through None unchanged. - - Args: - path: A string, path-like object, or None. - - Returns: - A UPath instance, or None if the input was None. - """ + """Convert a string or path-like to UPath, or return None.""" if path is None: return None - return UPath(path) + return UPath(path) if not isinstance(path, UPath) else path diff --git a/mkdocs.yml b/mkdocs.yml index 37ec0a4..e7da5c8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -35,6 +35,7 @@ nav: - Installation: user_guide/installation.md - Key Concepts: user_guide/key_concepts.md - Metrics and Statistics: user_guide/metrics.md + - Quantile Confidence Intervals: user_guide/quantile_confidence.md - Connect to Endpoints: user_guide/connect_endpoints.md - Run Experiments: user_guide/run_experiments.md - API Reference: @@ -70,6 +71,7 @@ nav: - results: reference/results.md - runner: reference/runner.md - tokenizers: reference/tokenizers.md + - quantile_ci: reference/quantile_ci.md - utils: reference/utils.md repo_url: https://github.com/awslabs/llmeter repo_name: awslabs/llmeter diff --git a/tests/test_quantile_ci.py b/tests/test_quantile_ci.py new file mode 100644 index 0000000..7e7bb83 --- /dev/null +++ b/tests/test_quantile_ci.py @@ -0,0 +1,138 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for quantile confidence interval estimation.""" + + +from llmeter.quantile_ci import ( + _binom_cdf, + _binom_ppf, + _ci_coverage, + can_estimate_quantile, + quantile_ci, +) + + +class TestBinomCdf: + """Test the pure-Python binomial CDF implementation.""" + + def test_known_value_n10_p05(self): + """Binom(10, 0.5) CDF at k=5 is 0.623046875.""" + assert abs(_binom_cdf(5, 10, 0.5) - 0.623046875) < 1e-9 + + def test_boundary_k_negative(self): + assert _binom_cdf(-1, 10, 0.5) == 0.0 + + def test_boundary_k_equals_n(self): + assert _binom_cdf(10, 10, 0.5) == 1.0 + + def test_p_zero(self): + assert _binom_cdf(0, 5, 0.0) == 1.0 + assert _binom_cdf(-1, 5, 0.0) == 0.0 + + def test_p_one(self): + assert _binom_cdf(4, 5, 1.0) == 0.0 + assert _binom_cdf(5, 5, 1.0) == 1.0 + + def test_large_n_no_overflow(self): + """Ensure no overflow for n=5000.""" + result = _binom_cdf(4950, 5000, 0.99) + assert 0.0 <= result <= 1.0 + + +class TestBinomPpf: + """Test the inverse CDF.""" + + def test_median_n10_p05(self): + """Smallest k with CDF >= 0.5 for Binom(10, 0.5) is 5.""" + assert _binom_ppf(0.5, 10, 0.5) == 5 + + def test_known_value_n100_p09(self): + """ppf(0.025, 100, 0.9) should be around 84.""" + result = _binom_ppf(0.025, 100, 0.9) + assert 82 <= result <= 86 + + def test_boundary_q_zero(self): + assert _binom_ppf(0, 100, 0.5) == 0 + + def test_boundary_q_one(self): + assert _binom_ppf(1, 100, 0.5) == 100 + + +class TestCiCoverage: + """Test the coverage probability calculation.""" + + def test_full_range_n3_p05(self): + """[X_(1), X_(3)] for n=3, p=0.5 should cover 50% (not 95%).""" + # Coverage = P(0 <= X <= 1) = P(X=0) + P(X=1) = 0.125 + 0.375 = 0.5 + coverage = _ci_coverage(0, 2, 3, 0.5) + assert abs(coverage - 0.5) < 0.01 + + def test_n8_p05_covers_95(self): + """n=8, p50: indices [1, 7] should achieve >= 95% coverage.""" + coverage = _ci_coverage(1, 7, 8, 0.5) + assert coverage >= 0.95 + + +class TestCanEstimateQuantile: + """Test the reliability gate function.""" + + def test_p50_needs_at_least_8(self): + """p50 at 95% confidence requires n >= 8.""" + assert not can_estimate_quantile(7, 0.5) + assert can_estimate_quantile(8, 0.5) + + def test_p90_needs_many_samples(self): + """p90 at 95% confidence requires n >= 46.""" + assert not can_estimate_quantile(10, 0.9) + assert not can_estimate_quantile(30, 0.9) + assert can_estimate_quantile(50, 0.9) + + def test_p99_needs_hundreds(self): + """p99 at 95% confidence requires n >= 482.""" + assert not can_estimate_quantile(100, 0.99) + assert can_estimate_quantile(500, 0.99) + + def test_n_less_than_2(self): + assert not can_estimate_quantile(0, 0.5) + assert not can_estimate_quantile(1, 0.5) + + def test_lower_confidence_needs_fewer_samples(self): + """At 80% confidence, fewer samples are needed.""" + assert can_estimate_quantile(5, 0.5, confidence=0.80) + + +class TestQuantileCi: + """Test the main CI computation function.""" + + def test_returns_none_for_small_sample(self): + """Should return None when sample is too small.""" + assert quantile_ci([1, 2, 3], 0.99) is None + + def test_returns_tuple_for_large_sample(self): + """Should return a valid (lower, upper) tuple.""" + import random + + random.seed(42) + data = [random.gauss(100, 10) for _ in range(500)] + ci = quantile_ci(data, 0.9) + assert ci is not None + lower, upper = ci + assert lower < upper + + def test_ci_brackets_point_estimate(self): + """The point estimate should typically fall within the CI.""" + import random + + random.seed(42) + data = [random.gauss(100, 10) for _ in range(500)] + ci = quantile_ci(data, 0.5) + assert ci is not None + point_est = sorted(data)[250] + assert ci[0] <= point_est <= ci[1] + + def test_empty_data(self): + assert quantile_ci([], 0.5) is None + + def test_single_value(self): + assert quantile_ci([42.0], 0.5) is None diff --git a/tests/test_utils_ci_gating.py b/tests/test_utils_ci_gating.py new file mode 100644 index 0000000..31e7d22 --- /dev/null +++ b/tests/test_utils_ci_gating.py @@ -0,0 +1,117 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for CI-gated quantile behavior in summary_stats_from_list.""" + +import random + + +from llmeter.utils import summary_stats_from_list + + +class TestSummaryStatsDefaultBehavior: + """Test that the default (confidence=None) preserves legacy behavior.""" + + def test_default_always_reports_all_quantiles(self): + """With default settings, all percentiles are always reported.""" + random.seed(42) + data = [random.gauss(100, 10) for _ in range(5)] + stats = summary_stats_from_list(data, percentiles=(50, 90, 99)) + assert "p50" in stats + assert "p90" in stats + assert "p99" in stats + + def test_default_no_ci_keys(self): + """Default behavior does not include CI bounds.""" + random.seed(42) + data = [random.gauss(100, 10) for _ in range(500)] + stats = summary_stats_from_list(data, percentiles=(50, 90, 99)) + assert "p90_ci_lower" not in stats + assert "p90_ci_upper" not in stats + + def test_empty_data_returns_empty(self): + stats = summary_stats_from_list([]) + assert stats == {} + + def test_single_value(self): + """Single value always reports the value for all percentiles.""" + stats = summary_stats_from_list([42.0], percentiles=(50, 90, 99)) + assert stats["p50"] == 42.0 + assert stats["p90"] == 42.0 + assert stats["p99"] == 42.0 + + +class TestSummaryStatsWithConfidence: + """Test opt-in confidence interval and gating behavior.""" + + def test_small_sample_omits_p99(self): + """With 20 samples and confidence=0.95, p99 should be omitted (needs ~473).""" + random.seed(42) + data = [random.gauss(100, 10) for _ in range(20)] + stats = summary_stats_from_list( + data, percentiles=(50, 90, 99), confidence=0.95 + ) + assert "average" in stats + assert "p99" not in stats + # p50 needs n >= 8, so should be present for n=20 + assert "p50" in stats + + def test_small_sample_omits_p90(self): + """With 20 samples, p90 also omitted (needs ~46).""" + random.seed(42) + data = [random.gauss(100, 10) for _ in range(20)] + stats = summary_stats_from_list( + data, percentiles=(50, 90, 99), confidence=0.95 + ) + assert "p90" not in stats + + def test_large_sample_includes_all(self): + """With 500 samples, all quantiles should be present.""" + random.seed(42) + data = [random.gauss(100, 10) for _ in range(500)] + stats = summary_stats_from_list( + data, percentiles=(50, 90, 99), confidence=0.95 + ) + assert "p50" in stats + assert "p90" in stats + assert "p99" in stats + + def test_ci_bounds_included(self): + """CI bounds should be attached when quantile is reported.""" + random.seed(42) + data = [random.gauss(100, 10) for _ in range(500)] + stats = summary_stats_from_list( + data, percentiles=(50, 90, 99), confidence=0.95 + ) + assert "p90_ci_lower" in stats + assert "p90_ci_upper" in stats + assert stats["p90_ci_lower"] < stats["p90"] + assert stats["p90_ci_upper"] > stats["p90"] + + def test_ci_lower_confidence_is_less_strict(self): + """Lower confidence level requires fewer samples.""" + random.seed(42) + data = [random.gauss(100, 10) for _ in range(30)] + # At 95%, p90 is gated (needs ~46) + stats_95 = summary_stats_from_list( + data, percentiles=(90,), confidence=0.95 + ) + assert "p90" not in stats_95 + + # At 80%, fewer samples needed + stats_80 = summary_stats_from_list( + data, percentiles=(90,), confidence=0.80 + ) + assert "p90" in stats_80 + + def test_single_value_gated_out(self): + """Single value: n=1 is too small for any quantile CI at 95%.""" + stats = summary_stats_from_list( + [42.0], percentiles=(50, 90, 99), confidence=0.95 + ) + # n=1 can't form any CI + assert "p50" not in stats + assert "p90" not in stats + assert "p99" not in stats + # average is always reported + assert "average" in stats