diff --git a/docs/early_stopping.md b/docs/early_stopping.md index 8a0c5930..59a2c485 100644 --- a/docs/early_stopping.md +++ b/docs/early_stopping.md @@ -17,7 +17,10 @@ estimate stays honest, which is exactly the regime edge/T2V workloads live in. For each target percentile `p` — every entry of the series' own report percentile grid at or above the median (`es_targets_from_grid`; the default grid yields p50/p75/p80/p90/p95/p97/p99/p99.9) — -at confidence `c = 0.99`, over the `n` ascending-sorted latencies of a series: +at confidence `c = 0.99`, over the `n` ascending-sorted latencies of a series +(percentiles use the grid convention — 0-100 — at every surface; only the LoadGen-parity +kernel keeps LoadGen's fraction domain, converted exactly once and unrounded, the same `/100` +`np.percentile` applies internally): ``` estimate = sorted[n - t], t = max{ i : n >= find_min_passing(i, p, d, c) + i } @@ -35,7 +38,7 @@ lowering `c` or raising `d` weakens the certified claim (`d > 0` certifies perce The pure math keeps defaulted arguments for parity tests only. **The percentile targets are not a separate list either** — they derive from the series' report -percentile grid, filtered to `p ≥ 0.5` (`ES_MIN_PERCENTILE`): the estimate is a _tail_ +percentile grid, filtered to `≥ p50` (`ES_MIN_PERCENTILE = 50.0`, grid convention): the estimate is a _tail_ certification (a conservative upper confidence bound), so below-median grid entries are skipped. One source of truth: whatever percentiles a series reports, ES covers — every scenario's gate percentile (p99 Server, p90 SingleStream/T2V) is always included, with nothing to tune. @@ -44,6 +47,65 @@ Each estimate is a _marginal_ `c`-confidence statement per percentile. Reporting fine for diagnostics, but a joint gate across all of them holds at lower than `c` confidence (multiple testing) — compliance gates should use the single scenario percentile. +## How the estimate is computed + +The statistical question: for a candidate bound `B`, can we claim "the true p-percentile is `<= B`" +at confidence `c`? Each sample is a Bernoulli trial — over or under `B`. If the true p-percentile +actually exceeded `B`, samples would land over `B` at a rate above `1 - p`, so observing few +over-latency samples in a long run is evidence for the bound. LoadGen accepts `B` when + +``` +P( <= t over-latency among n samples | over-latency rate = 1 - p ) < 1 - c +``` + +— a false certification slips through with probability below `1 - c = 1%`. That binomial tail has a +closed form as a regularized incomplete beta, `I_p(n - t, t + 1)`, evaluated with the Numerical +Recipes `betai` continued fraction (no scipy; numerically equal to LoadGen's Gauss-hypergeometric +form but converging in tens of iterations — see the docstrings in `metrics/early_stopping.py`). + +The reported estimate takes `B` to be an actual sample — the t-th highest — with `t` pushed as low +into the tail as the test allows. Both searches exploit monotonicity (exponential bracketing + +binary search), so the whole computation is `O(log² n)` beta evaluations on the sorted array: + +``` +odds(h, t) = P(<= t over-latency among h + t trials | rate 1 - p) # = I_p(h, t + 1) +find_min_passing(t) = min h with odds(h, t) < 1 - c # under-latency samples needed to absorb t +floor = find_min_passing(1) + 1 # smallest certifiable n (see cheat sheet) + +estimate(sorted, p): + if n < floor: return None # too few samples for any claim + t = largest t >= 1 with find_min_passing(t) + t <= n # the run's over-latency budget + return sorted[n - t] # t-th highest sample; t - 1 sit above it +``` + +Worked example (n = 10,000, p99): the floor is 662, the budget resolves to `t = 77`, and the +estimate is the 77th-highest sample — 76 samples sit strictly above it (one budget slot is spent on +the estimate itself, which keeps the claim strict). At the floor the budget is `t = 1` and the +estimate is the maximum observed sample — honest but maximally conservative; as `n` grows, +`t/n -> 1 - p` and the estimate converges onto the empirical percentile from above. + +## Cheat sheet: minimum samples per percentile + +The floor is `find_min_passing(1, p) + 1` at confidence 0.99 — the smallest run that can +certify percentile p at all. Below it the map reports `null` for that percentile (the run +"does not meet the standard" for that gate); at or above it the estimate is a valid +c = 0.99 upper confidence bound. + +| percentile | minimum samples | +| ---------- | --------------- | +| p50 | 11 | +| p75 | 24 | +| p80 | 31 | +| p90 | 64 | +| p95 | 130 | +| p97 | 219 | +| p99 | 662 | +| p99.9 | 6,636 | +| p99.99 | 66,381 | + +Rule of thumb: floor ≈ 6.64 / (1 − p/100) — one more "9" costs 10× the samples. Contrast +with the fixed-sample regime the feature replaces (~270k queries for Server p99). + ## Layering (who owns what) ``` diff --git a/scripts/early_stopping_estimate_from_events.py b/scripts/early_stopping_estimate_from_events.py index 196d3355..3819112c 100644 --- a/scripts/early_stopping_estimate_from_events.py +++ b/scripts/early_stopping_estimate_from_events.py @@ -36,7 +36,7 @@ # early_stopping_percentiles maps — exactly the # shape an ES-enabled run would have produced [--tokenizer ] # enables TPOT - [--percentiles 0.5,0.9,0.95,0.99] [--confidence 0.99] # stdout-analysis overrides + [--percentiles 50,90,95,99.9] [--confidence 0.99] # stdout-analysis overrides """ from __future__ import annotations @@ -65,7 +65,6 @@ CONFIDENCE, es_percentile_estimate, es_targets_from_grid, - grid_percentile_key, ) from inference_endpoint.metrics.report import ( SERIES_TO_SUMMARY_FIELD, @@ -257,10 +256,11 @@ def main(argv=None): ) ap.add_argument( "--percentiles", - default=",".join( - str(f) for f in es_targets_from_grid(DEFAULT_PERCENTILES).values() + default=",".join(es_targets_from_grid(DEFAULT_PERCENTILES)), + help=( + "stdout-analysis override, grid convention (0-100, e.g. 50,90,99.9); " + "--json always uses the summary grid" ), - help="stdout-analysis override (fractions); --json always uses the summary grid", ) ap.add_argument( "--confidence", @@ -274,11 +274,22 @@ def main(argv=None): raise SystemExit( "FATAL: --json requires --summary (the augmented output IS the summary)" ) - fallback_fractions = [float(x) for x in args.percentiles.split(",")] - if not all(0.0 < f < 1.0 for f in fallback_fractions): + values = {float(tok) for tok in args.percentiles.split(",")} + if not all(0.0 < v < 100.0 for v in values): + raise SystemExit( + f"FATAL: percentiles use the grid convention and must be in (0, 100), " + f"got {sorted(values)}" + ) + if any(v < 1.0 for v in values): + # a fraction-style value (pre-#423 convention) silently means a sub-1% + # percentile here — never a legitimate tail-certification target + bad = sorted(v for v in values if v < 1.0) raise SystemExit( - f"FATAL: percentiles must be in (0, 1), got {fallback_fractions}" + f"FATAL: percentiles now use the grid convention (0-100); {bad} would " + f"mean sub-1% percentiles — did you mean {[v * 100 for v in bad]}?" ) + # canonical float-style keys, deduped on the parsed value ("99" == "99.0") + fallback = {str(v): v for v in sorted(values, reverse=True)} if not 0.0 < args.confidence < 1.0: raise SystemExit(f"FATAL: confidence must be in (0, 1), got {args.confidence}") @@ -313,10 +324,7 @@ def main(argv=None): # the run's own grid: exact key strings, exact (descending) order targets = es_targets_from_grid(grid.keys()) else: - targets = { - grid_percentile_key(f): f - for f in sorted(fallback_fractions, reverse=True) - } + targets = fallback results = { key: es_percentile_estimate(values, f, args.confidence) for key, f in targets.items() @@ -332,11 +340,10 @@ def main(argv=None): f"empirical={_fmt(r.empirical, unit, div)} " f"estimate={_fmt(r.estimate, unit, div)}" ) - if r.estimate is not None and r.empirical: + if r.estimate is not None and r.empirical is not None: gap = r.estimate - r.empirical - line += ( - f" gap=+{_fmt(gap, unit, div)} (+{100 * gap / r.empirical:.2f}%)" - ) + pct = f" (+{100 * gap / r.empirical:.2f}%)" if r.empirical else "" + line += f" gap=+{_fmt(gap, unit, div)}{pct}" print(line) rows.append((field, key, r, unit, div)) if summary is not None: @@ -357,9 +364,10 @@ def main(argv=None): print("| metric | p | n | empirical | ES-adjusted | gap |") print("|---|---|---|---|---|---|") for field, key, r, unit, div in rows: - if r.estimate is not None and r.empirical: + if r.estimate is not None and r.empirical is not None: gap = r.estimate - r.empirical - g = f"+{_fmt(gap, unit, div)} (+{100 * gap / r.empirical:.2f}%)" + pct = f" (+{100 * gap / r.empirical:.2f}%)" if r.empirical else "" + g = f"+{_fmt(gap, unit, div)}{pct}" else: g = "-" print( diff --git a/src/inference_endpoint/async_utils/services/metrics_aggregator/registry.py b/src/inference_endpoint/async_utils/services/metrics_aggregator/registry.py index bd79a27a..acbb10f0 100644 --- a/src/inference_endpoint/async_utils/services/metrics_aggregator/registry.py +++ b/src/inference_endpoint/async_utils/services/metrics_aggregator/registry.py @@ -45,7 +45,6 @@ EarlyStoppingSpec, es_percentile_estimate, es_targets_from_grid, - grid_percentile_key, ) from .snapshot import ( @@ -242,11 +241,22 @@ def _es_estimates(self, sorted_values) -> dict[str, float | None] | None: if self._es_spec is None: return None spec = self._es_spec + try: + return self._compute_es_estimates(sorted_values, spec) + except Exception: + # Best-effort by design: this runs inside publish_final's + # build_snapshot, where an exception would cost the ENTIRE final + # report (the publisher's finalized guard makes retries no-ops). + logger.exception( + "%s: early-stopping computation failed; omitting the map", self.name + ) + return None + + def _compute_es_estimates( + self, sorted_values, spec: EarlyStoppingSpec + ) -> dict[str, float | None]: if spec.percentiles is not None: # explicit override: tests / offline analysis - targets = { - grid_percentile_key(f): f - for f in sorted(spec.percentiles, reverse=True) - } + targets = {str(v): float(v) for v in sorted(spec.percentiles, reverse=True)} else: targets = es_targets_from_grid(self._percentiles) results = { diff --git a/src/inference_endpoint/async_utils/services/metrics_aggregator/snapshot.py b/src/inference_endpoint/async_utils/services/metrics_aggregator/snapshot.py index aec6a40f..5c5691d2 100644 --- a/src/inference_endpoint/async_utils/services/metrics_aggregator/snapshot.py +++ b/src/inference_endpoint/async_utils/services/metrics_aggregator/snapshot.py @@ -120,7 +120,7 @@ class SeriesStat( percentiles: dict[str, float] histogram: list[tuple[tuple[float, float], int]] # Early-stopping percentile estimates (COMPLETE snapshots only, when enabled): - # compact {grid_percentile_key: estimate-or-None} map whose keys mirror + # compact {grid key (str of the grid entry): estimate-or-None} map mirroring # ``percentiles``; None value = insufficient samples for that percentile. # Optional trailing field so the array_like wire format stays # backward-compatible. None field = not computed. diff --git a/src/inference_endpoint/metrics/early_stopping.py b/src/inference_endpoint/metrics/early_stopping.py index 424f92d7..4baaecce 100644 --- a/src/inference_endpoint/metrics/early_stopping.py +++ b/src/inference_endpoint/metrics/early_stopping.py @@ -14,6 +14,13 @@ The regularized incomplete beta uses a continued fraction (Numerical Recipes ``betai``); it equals LoadGen's Gauss-hypergeometric ``beta_regularized`` but converges in tens of iterations. + +Percentile convention: every product surface (spec, estimates, grid targets, report keys, +script CLI) speaks the report-grid convention — 0-100, e.g. ``99.9`` — matching +``DEFAULT_PERCENTILES``, ``np.percentile``, and the summary's ``percentiles`` keys. Only the +LoadGen-parity kernel (``find_min_passing`` / ``_odds`` / ``_discard_count``) keeps LoadGen's +own fraction domain (0-1); the single conversion happens inside ``es_percentile_estimate``, +unrounded — the same ``/100`` ``np.percentile`` applies internally, so grid parity is exact. """ from __future__ import annotations @@ -32,7 +39,6 @@ "es_percentile_estimate", "es_targets_from_grid", "find_min_passing", - "grid_percentile_key", ] # LoadGen hardcodes confidence c = 0.99 and tolerance d = 0.0 (results.cc:157-158). They are @@ -45,53 +51,40 @@ # ES runs for every percentile the series' report grid shows at or above the median. The # estimate is a tail certification (a conservative upper confidence bound), so below-median # grid entries (p25/p10/...) are not meaningful gates and are skipped. Deriving from the grid -# keeps one source of truth: whatever percentiles a series reports, ES covers. -ES_MIN_PERCENTILE: Final[float] = 0.5 +# keeps one source of truth: whatever percentiles a series reports, ES covers. Grid +# convention (0-100), like every percentile surface in this module. +ES_MIN_PERCENTILE: Final[float] = 50.0 def es_targets_from_grid(grid_percentiles: Iterable[float | str]) -> dict[str, float]: - """{report-grid key: fraction} for every grid entry in the ES domain. + """{report-grid key: percentile} for every grid entry in the ES domain. Keys are ``str()`` of the ORIGINAL grid entries and the map preserves the grid's own order (descending in the default grid), so the compact map - overlays the ``percentiles`` grid 1:1 in both naming and ordering. Accepts - the entries as numbers (registry grids) or strings (summary-JSON keys — an - int grid entry ``99`` keys as ``"99"`` either way). Entries below the median + overlays the ``percentiles`` grid 1:1 in naming and ordering. Accepts the + entries as numbers (registry grids) or strings (summary-JSON keys — an int + grid entry ``99`` keys as ``"99"`` either way). Values stay in the grid + convention (0-100) — no conversion, no rounding. Entries below the median are skipped (an ES estimate is a tail certification) and ``>= 100`` is - excluded — p1.0 is outside the binomial test's domain and would otherwise - crash the terminal snapshot. Fractions are rounded to 6 decimals: lossless - for grid values while avoiding float-division artifacts (99.9/100 != 0.999). + excluded — p100 is outside the binomial test's domain and would otherwise + crash the terminal snapshot. """ targets: dict[str, float] = {} for p in grid_percentiles: - # 6 fraction decimals == grid_percentile_key's 4 percent decimals — same - # precision, different spaces. - fraction = round(float(p) / 100.0, 6) - if ES_MIN_PERCENTILE <= fraction < 1.0: - # str(p) of the ORIGINAL entry, deliberately NOT grid_percentile_key: - # reconstructing the key from the fraction would render float-style - # ("99.0") and break the 1:1 overlay for int grids keyed as "99". - targets[str(p)] = fraction + value = float(p) + if ES_MIN_PERCENTILE <= value < 100.0: + targets[str(p)] = value return targets -def grid_percentile_key(p: float) -> str: - """Report-grid-style key for a fraction percentile: 0.5 -> "50.0", 0.999 -> "99.9". - - Used for explicit percentile overrides (tests / offline analysis), where no grid - string exists to carry through; grid-derived targets keep the grid's own keys via - ``es_targets_from_grid``. ``round`` absorbs float artifacts (0.97 * 100 != 97.0). - """ - return str(round(p * 100.0, 4)) - - @dataclass(frozen=True, slots=True) class EarlyStoppingSpec: """Resolved early-stopping parameters used by the aggregator. ``percentiles=None`` (the default) derives the target set from each series' own - report grid via ``es_targets_from_grid``; explicit tuples are for tests and - offline analysis only — the config surface is a single ``enabled`` flag. + report grid via ``es_targets_from_grid``; explicit tuples (grid convention, + 0-100) are for tests and offline analysis only — the config surface is a + single ``enabled`` flag. """ percentiles: tuple[float, ...] | None = None @@ -113,13 +106,15 @@ def _betacf(a: float, b: float, x: float) -> float: guard against division blow-up); any positive value far below the smallest normal double (~2.2e-308) times a typical term works. - ``400``: iteration cap. For the boundary searches at very large n (a ~ b ~ n/2 - with x within O(1/sqrt(n)) of the mean — the p50/p75 targets at tens of millions - of samples) Lentz needs ~O(sqrt(a)) iterations and CAN hit this cap; the running - product has plateaued by then (measured <= 3e-8 relative error vs - scipy.special.betainc at n=1e7, far below the 0.01 threshold the result is - compared against), so the truncated value is returned rather than raising — - this runs on the terminal-snapshot path, where an exception would cost the - whole final report. + with x near the mean — the p50/p75 targets at tens of millions of samples) + Lentz needs ~O(sqrt(a)) iterations and CAN hit this cap, returning a truncated + value whose error near the distribution midpoint grows with n (order 1e-4 at + a = b = 5e6 and worse beyond — do NOT reuse ``_betai`` standalone where + midpoint accuracy matters). Truncation is safe HERE because every decision + compares odds against 1 - c (~0.01), far from the slow-converging midpoint: + ``find_min_passing`` returns identical answers with cap 400 vs 20000 on the + LoadGen references (459/44/661). Returned rather than raised — this runs on + the terminal-snapshot path, where an exception would cost the final report. """ eps, fpmin = 3e-16, 1e-300 qab, qap, qam = a + b, a + 1.0, a - 1.0 @@ -209,17 +204,18 @@ def find_min_passing( LoadGen's ``MinPassingQueriesFinder`` (``loadgen/early_stopping.cc:62-114``): the smallest number of under-latency queries that, alongside ``t`` over-latency queries, lets you conclude at confidence ``c`` that the true p-percentile meets the bound. - ``_odds`` is monotonically decreasing in ``h``, so exponential bracketing (double - ``hi`` until it passes) followed by binary search finds the boundary exactly. + The pass test is strict (``odds < 1 - c``), matching LoadGen's boundary handling + (``early_stopping.cc:70``). ``_odds`` is monotonically decreasing in ``h``, so + exponential bracketing followed by binary search finds the boundary exactly. """ _validate_domain(p, d, c) target = 1.0 - c lo, hi = 1, 2 - while _odds(hi, t, p, d) > target: + while _odds(hi, t, p, d) >= target: hi *= 2 while lo < hi: mid = (lo + hi) // 2 - if _odds(mid, t, p, d) <= target: + if _odds(mid, t, p, d) < target: hi = mid else: lo = mid + 1 @@ -230,9 +226,9 @@ def _discard_count(n: int, p: float, d: float, c: float) -> int: """Largest ``t >= 1`` with ``n >= find_min_passing(t) + t``; 0 below the floor. This is the LoadGen SingleStream estimate construction (``results.cc:162-226``): - discard the ``t`` highest samples such that the run of ``n`` queries would still - pass the binomial test with those as its over-latency set — the (t+1)-th highest - value is then a c-confidence upper bound on the true p-percentile. Same + find the largest over-latency budget ``t`` the run of ``n`` queries can still + absorb — the t-th highest value is then a c-confidence upper bound on the true + p-percentile, with the ``t - 1`` samples above it discarded. Same exponential-bracket + binary-search shape as ``find_min_passing`` (the passing margin ``n - (find_min_passing(t) + t)`` is monotone in ``t``). """ @@ -250,6 +246,7 @@ def _discard_count(n: int, p: float, d: float, c: float) -> int: return lo +@dataclass(frozen=True, slots=True) class EarlyStoppingResult: """Result of an early-stopping percentile estimate. @@ -259,33 +256,13 @@ class EarlyStoppingResult: itself is the ``discarded + 1``-th highest sample). """ - __slots__ = ( - "percentile", - "confidence", - "n", - "estimate", - "empirical", - "min_queries", - "discarded", - ) - - def __init__( - self, - percentile: float, - confidence: float, - n: int, - estimate: float | None, - empirical: float | None, - min_queries: int, - discarded: int, - ) -> None: - self.percentile = percentile - self.confidence = confidence - self.n = n - self.estimate = estimate - self.empirical = empirical - self.min_queries = min_queries - self.discarded = discarded + percentile: float + confidence: float + n: int + estimate: float | None + empirical: float | None + min_queries: int + discarded: int def as_dict(self) -> dict[str, float | int | bool | None]: return { @@ -305,19 +282,29 @@ def es_percentile_estimate( percentile: float, confidence: float = CONFIDENCE, ) -> EarlyStoppingResult: - """Conservative early-stopping percentile estimate over an ascending-sorted latency series.""" + """Conservative early-stopping estimate over an ascending-sorted latency series. + + ``percentile`` uses the grid convention (0-100, e.g. ``99.9``) — a value below + 1 almost certainly means a caller still passing the pre-#423 fraction style + and will be interpreted as a sub-1% percentile. The single conversion to the + kernel's fraction domain happens here, unrounded — the same ``/100`` + ``np.percentile`` applies internally, so the ``empirical`` value can never + diverge from the report grid's ``method="lower"`` entry. + """ + if not 0.0 < percentile < 100.0: + raise ValueError(f"percentile must be in (0, 100), got {percentile}") + fraction = percentile / 100.0 n = len(sorted_latencies) - min_queries = find_min_passing(1, percentile, TOLERANCE, confidence) + 1 + min_queries = find_min_passing(1, fraction, TOLERANCE, confidence) + 1 # Same order statistic as the report's percentile grid (np.percentile with - # method="lower": index floor(p*(n-1))) so the block's empirical can never - # disagree with the grid value in the same result_summary.json. - emp_idx = int(percentile * (n - 1)) if n else 0 + # method="lower": index floor(p*(n-1))). + emp_idx = int(fraction * (n - 1)) if n else 0 empirical = sorted_latencies[emp_idx] if n else None if n < min_queries: return EarlyStoppingResult( percentile, confidence, n, None, empirical, min_queries, 0 ) - t = _discard_count(n, percentile, TOLERANCE, confidence) + t = _discard_count(n, fraction, TOLERANCE, confidence) # The estimate is the t-th highest sample, so the number of samples actually # discarded (strictly above the reported value) is t - 1; at the floor # (t == 1) nothing is discarded. diff --git a/src/inference_endpoint/metrics/report.py b/src/inference_endpoint/metrics/report.py index ac561570..e6b0150c 100644 --- a/src/inference_endpoint/metrics/report.py +++ b/src/inference_endpoint/metrics/report.py @@ -275,9 +275,10 @@ def _counter(key: str) -> int: def _series_dict(key: str) -> dict[str, Any]: stat = series.get(key) - if stat is None or stat.get("count", 0) == 0: - return {} - return _series_to_metric_dict(stat) + # count==0 handling lives in _series_to_metric_dict, which preserves + # the all-null early_stopping_percentiles map for enabled-but-empty + # series — short-circuiting here would silently drop it. + return _series_to_metric_dict(stat) if stat is not None else {} version_info = get_version_info() raw_duration_ns = _counter("tracked_duration_ns") diff --git a/tests/unit/async_utils/services/metrics_aggregator/test_early_stopping_integration.py b/tests/unit/async_utils/services/metrics_aggregator/test_early_stopping_integration.py index 72c2ec21..bc9a7bb4 100644 --- a/tests/unit/async_utils/services/metrics_aggregator/test_early_stopping_integration.py +++ b/tests/unit/async_utils/services/metrics_aggregator/test_early_stopping_integration.py @@ -140,6 +140,28 @@ def test_custom_grid_with_p100_and_int_keys_is_safe(): assert list(esp) == ["99", "50"] # 100.0 excluded; int keys + grid order kept +def test_report_from_snapshot_preserves_empty_series_map(): + # The HIGH round-3 finding: _series_to_metric_dict preserved the map but + # Report.from_snapshot._series_dict short-circuited count==0 first, so the + # summary still dropped it. Pin the REAL consumer path, not the helper. + from inference_endpoint.metrics.report import Report + + report = Report.from_snapshot(_snap(_registry(EarlyStoppingSpec(), n=0))) + esp = report.ttft["early_stopping_percentiles"] + assert list(esp) == _GRID_ES_KEYS + assert all(v is None for v in esp.values()) + + +def test_es_failure_does_not_kill_the_final_snapshot(): + # ES runs inside publish_final's build_snapshot, where an exception would + # cost the entire final report — a bad explicit override must degrade to + # "map omitted", never raise. + reg = _registry(EarlyStoppingSpec(percentiles=(150.0,)), n=100) # out of domain + d = _snap(reg) + assert "early_stopping_percentiles" not in _series(d, "ttft_ns") + assert _series(d, "ttft_ns")["percentiles"] # exact stats still intact + + def test_repeated_complete_snapshots_are_identical(): # The exact path sorts the raw array IN PLACE (avoids a full transient copy); # a second COMPLETE snapshot must be byte-identical or the mutation leaked. diff --git a/tests/unit/metrics/test_early_stopping.py b/tests/unit/metrics/test_early_stopping.py index fdbcc168..f3702b50 100644 --- a/tests/unit/metrics/test_early_stopping.py +++ b/tests/unit/metrics/test_early_stopping.py @@ -17,7 +17,6 @@ es_percentile_estimate, es_targets_from_grid, find_min_passing, - grid_percentile_key, ) pytestmark = pytest.mark.unit @@ -41,7 +40,7 @@ def test_constants_and_spec_defaults(): # default to grid derivation (percentiles=None), not a private list. assert CONFIDENCE == 0.99 assert TOLERANCE == 0.0 - assert ES_MIN_PERCENTILE == 0.5 + assert ES_MIN_PERCENTILE == 50.0 # grid convention (0-100) spec = EarlyStoppingSpec() assert spec.percentiles is None and spec.confidence == CONFIDENCE assert not hasattr(spec, "tolerance") @@ -49,13 +48,13 @@ def test_constants_and_spec_defaults(): def test_grid_derivation_and_key_format(): # The ES targets and map keys must overlay the report's percentile grid 1:1 - # (keys are str() of the ORIGINAL grid values — int grids key as "99"), and - # out-of-domain entries must be excluded: a grid containing 100.0 would - # otherwise crash the terminal snapshot (default-on!), below-median entries - # are not tail certifications. + # (keys are str() of the ORIGINAL grid values — int grids key as "99"), in + # the grid's own order, with values staying in the grid convention (0-100, + # no conversion or rounding). Out-of-domain entries are excluded: 100.0 + # would crash the terminal snapshot (default-on!), below-median entries are + # not tail certifications. grid = (100.0, 99.9, 99.0, 97.0, 95.0, 90.0, 80.0, 75.0, 50.0, 25.0, 1.0) targets = es_targets_from_grid(grid) - # grid ORDER preserved (descending in the default grid), 100.0/below-median excluded assert list(targets) == [ "99.9", "99.0", @@ -66,27 +65,16 @@ def test_grid_derivation_and_key_format(): "75.0", "50.0", ] - assert ( - targets["99.9"] == 0.999 and targets["97.0"] == 0.97 - ) # float artifacts absorbed - assert list(es_targets_from_grid((99, 90, 50))) == [ - "99", - "90", - "50", - ] # int keys kept - assert list(es_targets_from_grid(("99.0", "50.0"))) == [ - "99.0", - "50.0", - ] # str keys kept - # explicit-spec overrides still key float-style via grid_percentile_key - assert grid_percentile_key(0.999) == "99.9" + assert targets["99.9"] == 99.9 and targets["50.0"] == 50.0 # values as given + assert list(es_targets_from_grid((99, 90, 50))) == ["99", "90", "50"] + assert list(es_targets_from_grid(("99.0", "50.0"))) == ["99.0", "50.0"] def test_estimate_reference_values_and_conservatism(): # Anchors the end-to-end estimate on a known input: discard count, floor, and # the invariant that the estimate never under-reports the empirical value. arr = [float(i) for i in range(10000)] - r = es_percentile_estimate(arr, 0.99) + r = es_percentile_estimate(arr, 99.0) # t=77: the estimate is the 77th-highest sample; 76 samples sit above it assert (r.discarded, r.min_queries) == (76, 662) assert r.estimate == arr[10000 - 77] @@ -95,35 +83,52 @@ def test_estimate_reference_values_and_conservatism(): def test_sufficiency_floor_boundary(): # Below the floor the estimate must be None (never a fabricated bound), the - # empirical value must still be reported, and n=0 must not crash. - below = es_percentile_estimate([float(i) for i in range(600)], 0.99) # < 662 + # empirical value must still be reported, and n=0 must not crash. Just above + # the floor the budget is t=1: nothing is discarded and the estimate IS the + # maximum observed sample. + below = es_percentile_estimate([float(i) for i in range(600)], 99.0) # < 662 assert below.estimate is None and below.empirical is not None - assert ( - es_percentile_estimate([float(i) for i in range(700)], 0.99).estimate - is not None - ) - empty = es_percentile_estimate([], 0.99) + arr = [float(i) for i in range(663)] + at_floor = es_percentile_estimate(arr, 99.0) + assert at_floor.estimate == arr[-1] and at_floor.discarded == 0 + empty = es_percentile_estimate([], 99.0) assert empty.estimate is None and empty.empirical is None and empty.n == 0 +def test_betai_branches_and_cap_regime(): + # Pin both evaluation branches against closed forms — I_x(a,1) = x^a exercises + # the direct branch, I_x(1,b) = 1-(1-x)^b at large x the reflection branch — + # and pin that the iteration-cap regime (a ~ b, midpoint x) stays sane and + # exception-free (accuracy there is deliberately NOT claimed; see _betacf). + from inference_endpoint.metrics.early_stopping import _betai + + assert abs(_betai(3.0, 1.0, 0.2) - 0.2**3) < 1e-12 # direct branch + assert abs(_betai(1.0, 3.0, 0.9) - (1.0 - 0.1**3)) < 1e-12 # reflection branch + assert 0.4 < _betai(5e6, 5e6, 0.5) < 0.6 # cap regime: truncated but sane + + def test_empirical_matches_report_grid_convention(): # `empirical` must use the grid's order statistic (np.percentile # method="lower": floor(p*(n-1))) or the block can contradict the grid value # in the same summary. n=50/p99 discriminates: floor=48 vs ceil(p*n)-1=49. arr = [float(i) for i in range(50)] - assert es_percentile_estimate(arr, 0.99).empirical == arr[48] + assert es_percentile_estimate(arr, 99.0).empirical == arr[48] def test_invalid_domain_raises(): # p >= 1 makes the doubling search non-terminating (a real hang, reproduced # in review); c >= 1 only exits via float underflow with a garbage result. for bad_call in ( + # kernel (fraction domain, loadgen parity) lambda: find_min_passing(1, 1.0), lambda: find_min_passing(1, 0.0), lambda: find_min_passing(1, 0.99, c=1.0), lambda: find_min_passing(1, 0.99, d=0.99), # tolerance must stay below p - lambda: es_percentile_estimate([1.0, 2.0], 1.5), - lambda: es_percentile_estimate([1.0, 2.0], 0.99, confidence=0.0), + # product surface (grid convention, 0-100) + lambda: es_percentile_estimate([1.0, 2.0], 150.0), + lambda: es_percentile_estimate([1.0, 2.0], 100.0), + lambda: es_percentile_estimate([1.0, 2.0], 0.0), + lambda: es_percentile_estimate([1.0, 2.0], 99.0, confidence=0.0), ): with pytest.raises(ValueError): bad_call() @@ -132,7 +137,7 @@ def test_invalid_domain_raises(): def test_result_dict_shape(): # The block dict is the post-hoc script's --json contract; tolerance is an # algorithm constant and must not be reported as if it were configuration. - d = es_percentile_estimate([float(i) for i in range(10000)], 0.99).as_dict() + d = es_percentile_estimate([float(i) for i in range(10000)], 99.0).as_dict() assert d["sufficient"] is True assert set(d) == { "percentile", diff --git a/tests/unit/scripts/test_early_stopping_estimate_from_events.py b/tests/unit/scripts/test_early_stopping_estimate_from_events.py index 2e2483d7..52147b43 100644 --- a/tests/unit/scripts/test_early_stopping_estimate_from_events.py +++ b/tests/unit/scripts/test_early_stopping_estimate_from_events.py @@ -215,6 +215,26 @@ def test_main_augments_summary_in_run_shape(tmp_path, capsys): capsys.readouterr() # drain +def test_fallback_percentiles_dedupe_on_value(tmp_path, capsys): + # "99" and "99.0" are the same target — one computation, one canonical key. + events = _write( + tmp_path, + [ + _ev("session.start_performance_tracking", 10), + _ev("sample.issued", 100, "a"), + _ev("sample.recv_first", 150, "a"), + _ev("sample.complete", 400, "a", _tmo(["x", "one two"])), + _ev("session.stop_performance_tracking", 500), + ], + ) + mod.main([str(events), "--percentiles", "99,99.0,90"]) + out = capsys.readouterr().out + # one row per metric section for the canonical key; a bare "p99:" row would + # mean "99" survived as a second target alongside "99.0" + assert out.count("p99.0:") == out.count("p90.0:") > 0 + assert "p99: " not in out + + def test_main_cli_contract(tmp_path, capsys): # invalid percentile/confidence must exit up front (they would hang or # corrupt the math); --json without --summary is a contract error. @@ -231,7 +251,10 @@ def test_main_cli_contract(tmp_path, capsys): with pytest.raises(SystemExit): mod.main([str(events), "--json", str(tmp_path / "o.json")]) with pytest.raises(SystemExit): - mod.main([str(events), "--percentiles", "1.0"]) + mod.main([str(events), "--percentiles", "150"]) # grid convention: (0, 100) + with pytest.raises(SystemExit): + # pre-#423 fraction style must hard-error, not silently mean p0.99% + mod.main([str(events), "--percentiles", "0.99"]) with pytest.raises(SystemExit): mod.main([str(events), "--confidence", "1.0"]) capsys.readouterr() # drain