fix(consensus): disable unsafe LPIPS early stopping

leoma/app/validator/main.py

@@ -525,6 +525,7 @@ async def _publish_dashboard(
                 "delta_threshold": SPEC.duel.delta_threshold,
                 "alpha": SPEC.duel.alpha,
                 "n_bootstrap": SPEC.duel.n_bootstrap,
+                "early_stop_enabled": SPEC.duel.early_stop_enabled,
                 "consensus_digest": CONSENSUS_DIGEST,
                 "corpus_pinned": SPEC.corpus.pinned,
             },

leoma/chain.toml

@@ -102,12 +102,14 @@ delta_threshold = 0.0025
 alpha = 0.001
 n_bootstrap = 10000
 base_seed = 0
-# Early stop fires once the challenger cannot reach delta_threshold even if every
-# remaining clip went as well as it possibly could. Expressed as a MULTIPLE of the
-# threshold so it tracks the metric's scale through any recalibration. At 20x it only
-# fires on a hopeless model and cannot touch a marginal contender — and since an early
-# stop can only ever yield verdict="king", a validator running with it still agrees
-# with one running without it.
+# Disabled for launch safety. Early stopping is only verdict-preserving when the
+# assumed per-clip advantage is a proven global upper bound for the pinned metric.
+# LPIPS has no established 0.05 bound, so the old 20x heuristic could reject a model
+# before later clips completed a legitimate comeback.
+early_stop_enabled = false
+# Retained as a pinned future setting. It has no effect while disabled. Do not enable
+# it based on an empirical percentile or observed maximum; first establish a true
+# metric bound and cover the equivalence with adversarial tests.
 early_stop_factor = 20.0
 # The freeze-baseline gate. A challenger must beat not only the king but a trivial
 # "hold the conditioning frame" cheat — CONFIDENTLY — or it is rejected even if it beat

leoma/eval/spec.py

@@ -21,7 +21,7 @@
 """
 from __future__ import annotations
 
-from typing import Literal
+from typing import Literal, Optional
 
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
@@ -100,9 +100,13 @@ class DuelSpec(_Pinned):
     alpha: float = Field(gt=0, lt=1)
     n_bootstrap: int = Field(ge=1)
     base_seed: int
-    #: Early stop fires at ``early_stop_factor × delta_threshold`` of best-possible
-    #: remaining advantage. Expressed as a multiple so it tracks the metric's scale
-    #: through any recalibration instead of being a magic constant.
+    #: Early stopping is consensus-visible because enabling it can change how many
+    #: clips are scored. It must remain disabled unless ``early_stop_factor`` is
+    #: backed by a proven global upper bound for the pinned metric's per-clip
+    #: advantage. An empirical quantile or a convenient multiple is not such a bound.
+    early_stop_enabled: bool
+    #: When enabled, each remaining clip is assumed able to contribute at most
+    #: ``early_stop_factor × delta_threshold`` of challenger advantage.
     early_stop_factor: float = Field(ge=0)
     #: The freeze-baseline gate's margin, as a fraction of the cheat's own mean score.
     #: Dimensionless on purpose — see chain.toml.
@@ -143,8 +147,10 @@ def digest(self) -> str:
         return digest_obj(self.model_dump(mode="json"))
 
     @property
-    def early_stop_max_advantage(self) -> float:
-        """The absolute early-stop bound, derived from the pinned multiple."""
+    def early_stop_max_advantage(self) -> Optional[float]:
+        """Return the pinned bound only when early stopping is explicitly enabled."""
+        if not self.duel.early_stop_enabled:
+            return None
         return self.duel.early_stop_factor * self.duel.delta_threshold
 
     def require_duel_ready(self) -> None:

tests/unit/test_consensus_spec.py

@@ -20,7 +20,8 @@ class TestNoDefaults:
     """A field with a default is a field a validator can silently forget."""
 
     @pytest.mark.parametrize("missing", ["metric", "metric_device", "n_clips",
-                                         "delta_threshold", "alpha", "n_bootstrap"])
+                                         "delta_threshold", "alpha", "n_bootstrap",
+                                         "early_stop_enabled"])
     def test_a_missing_duel_field_raises(self, missing):
         spec = pinned_spec()
         fields = spec.duel.model_dump()
@@ -120,36 +121,23 @@ def test_a_garbage_digest_is_rejected_outright(self):
 
 
 class TestEarlyStopBound:
-    def test_derived_from_the_threshold_so_it_tracks_recalibration(self):
+    def test_shipped_consensus_disables_unproven_bound(self):
         spec = pinned_spec()
-        assert spec.early_stop_max_advantage == pytest.approx(
-            spec.duel.early_stop_factor * spec.duel.delta_threshold
-        )
-
-    def test_at_20x_it_cannot_touch_a_marginal_challenger(self):
-        """Sanity-check the bound is loose enough to only kill hopeless models.
-
-        With 32 clips and delta=0.0025, early stop can only fire if the challenger
-        is already so far behind that even a perfect run of the remaining clips
-        can't reach the threshold. A challenger that is merely *slightly* worse
-        must always be scored to the end — otherwise the bound is silently changing
-        verdicts, not just saving GPU.
-        """
-        from leoma.eval.bootstrap import can_still_win
+        assert spec.duel.early_stop_enabled is False
+        assert spec.early_stop_max_advantage is None
 
+    def test_enabled_bound_is_derived_from_pinned_factor(self):
         spec = pinned_spec()
-        bound = spec.early_stop_max_advantage
-        n = spec.duel.n_clips
-
-        # Halfway in, and marginally behind: must keep going.
-        marginal = [-0.001] * (n // 2)
-        assert can_still_win(
-            [0.0] * (n // 2), [-d for d in marginal], remaining=n // 2,
-            delta_threshold=spec.duel.delta_threshold, best_possible_advantage=bound,
+        enabled = spec.model_copy(
+            update={"duel": spec.duel.model_copy(update={"early_stop_enabled": True})}
+        )
+        assert enabled.early_stop_max_advantage == pytest.approx(
+            enabled.duel.early_stop_factor * enabled.duel.delta_threshold
         )
 
-        # Halfway in, and hopelessly behind: stop.
-        assert not can_still_win(
-            [0.0] * (n // 2), [10.0] * (n // 2), remaining=n // 2,
-            delta_threshold=spec.duel.delta_threshold, best_possible_advantage=bound,
+    def test_enablement_changes_consensus_digest(self):
+        spec = pinned_spec()
+        enabled = spec.model_copy(
+            update={"duel": spec.duel.model_copy(update={"early_stop_enabled": True})}
         )
+        assert enabled.digest() != spec.digest()

tests/unit/test_video_runner.py

@@ -85,6 +85,37 @@ def test_early_stop_hopeless(self):
         assert v["early_stopped"] is True
         assert len(v["per_clip"]) < 20  # bailed early
 
+    def test_disabled_early_stop_preserves_a_late_comeback(self):
+        """An assumed bound can be exceeded; None must always score the full exam."""
+        clips = _clips(n=20)
+
+        def king(clip, _seed):
+            if clip.clip_index < 2:
+                return clip.truth_frames
+            return np.clip(clip.truth_frames.astype(float) + 90, 0, 255).astype("uint8")
+
+        def challenger(clip, _seed):
+            if clip.clip_index < 2:
+                return np.clip(clip.truth_frames.astype(float) + 90, 0, 255).astype("uint8")
+            return clip.truth_frames
+
+        complete = run_duel(
+            clips, king, challenger, mse, master_seed=3,
+            delta_threshold=0.0025, alpha=0.001, n_bootstrap=500,
+            early_stop_max_advantage=None,
+        )
+        unsafe = run_duel(
+            clips, king, challenger, mse, master_seed=3,
+            delta_threshold=0.0025, alpha=0.001, n_bootstrap=500,
+            early_stop_max_advantage=0.05,
+        )
+
+        assert complete["verdict"] == "challenger"
+        assert complete["early_stopped"] is False
+        assert len(complete["per_clip"]) == len(clips)
+        assert unsafe["verdict"] == "king"
+        assert unsafe["early_stopped"] is True
+
     def test_no_clips_raises(self):
         with pytest.raises(ValueError):
             run_duel([], _far, _near, mse, master_seed=1, delta_threshold=0.0025, alpha=0.001, n_bootstrap=10)