Generate king and challenger concurrently on separate devices (throughput, not consensus)

env.eval.example

@@ -17,5 +17,23 @@ LEOMA_MODEL_CACHE_DIR=/tmp/leoma/hippius_models
 R2_VIDEOS_READ_ACCESS_KEY=
 R2_VIDEOS_READ_SECRET_KEY=
 
+# Concurrent king/challenger generation — a THROUGHPUT setting, not a consensus one
+# (it is never sent by the validator and never hashed into consensus_digest). On a
+# multi-GPU box, generating both duelists at the same time on separate devices
+# roughly halves duel wall-clock. Falls back to today's sequential single-device
+# behavior if fewer than 2 CUDA devices are visible. See leoma/eval/devices.py.
+#
+# ⚠️ If you enable this, re-run `leoma calibrate` with it enabled — it changes what
+# noise delta_threshold has to absorb (see docs/TESTNET_RUNBOOK.md).
+# LEOMA_CONCURRENT_GENERATION=1
+
+# LEOMA_KING_DEVICE / LEOMA_CHALLENGER_DEVICE are independent of the concurrency flag
+# above: they also matter with LEOMA_CONCURRENT_GENERATION unset, e.g. when running
+# several eval-server PROCESSES on one multi-GPU box (one per EVAL_SERVER_URLS entry)
+# and pinning each process to its own device pair so their VRAM doesn't all pile onto
+# the same default GPU. Set both together, or neither — a single one is ignored.
+# LEOMA_KING_DEVICE=cuda:0
+# LEOMA_CHALLENGER_DEVICE=cuda:1
+
 # Optional deterministic prompt for the I2V pipeline (same for king + challenger).
 # LEOMA_DUEL_PROMPT=

leoma/eval/devices.py

@@ -0,0 +1,119 @@
+"""Which physical GPU each duelist generates on — a throughput knob, not a consensus one.
+
+Today king and challenger load onto the **same** default CUDA device and generate
+**sequentially**: king's clip, then the challenger's clip, one at a time. On an 8×H100
+box that wastes seven idle GPUs for the length of every duel. Since each clip is
+independent and both duelists use the same per-clip seed, king and challenger can
+generate on two *separate* devices at the same time — roughly halving duel wall-clock.
+
+**Why this is deliberately NOT part of the consensus surface** (``chain.toml`` /
+:class:`~leoma.eval.spec.ConsensusSpec`): generation was already conceded to be
+non-bit-exact across GPU *architectures* (see ``determinism.py`` and
+``eval/calibrate.py``), and ``delta_threshold`` exists precisely to absorb that noise
+regardless of its source. Whether two duelists happen to run on the same physical
+card or two different (but identical) cards in the same box is exactly that kind of
+generation-side noise, not a fact about the exam. So it is a per-box performance
+setting, resolved from the environment, exactly like ``metric_device``.
+
+**The one thing operators must not forget:** enabling this changes *what* noise
+``delta_threshold`` has to absorb. Even two identical H100s in one node are not
+provably bit-identical generators. If you turn this on, re-run
+``leoma calibrate`` with it enabled — you are calibrating against the runtime
+configuration you intend to use in production, not the one you happened to measure
+before you turned this on. See ``docs/TESTNET_RUNBOOK.md``.
+
+This module is a pure function of already-known inputs (feature flag, device count,
+operator overrides) so the *decision* is unit-testable with no GPU. The one line that
+needs an actual CUDA runtime — ``torch.cuda.device_count()`` — lives in the eval
+server's tiny, untested glue, not here.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class DuelDevices:
+    """Where each duelist should load. ``None`` means "today's default device"."""
+
+    king_device: Optional[str]
+    challenger_device: Optional[str]
+    #: True only when generation will ACTUALLY overlap. False whenever the request
+    #: for concurrency couldn't be honored (fewer than 2 devices, or a collision) —
+    #: callers branch on this, not on whether concurrency was merely *requested*.
+    concurrent: bool
+    #: Human-readable explanation of the decision, for logs and the verdict's audit block.
+    note: str
+
+
+def resolve_duel_devices(
+    *,
+    concurrent_enabled: bool,
+    cuda_device_count: int,
+    king_device_override: Optional[str] = None,
+    challenger_device_override: Optional[str] = None,
+) -> DuelDevices:
+    """Decide where king and challenger generate. Falls back to today's behavior
+    whenever concurrency can't be honored safely, rather than guessing.
+
+    ``king_device_override`` / ``challenger_device_override`` let an operator pin
+    specific devices (e.g. to dodge a card another process is using, or on a box
+    where ``torch.cuda.device_count()`` undercounts what's actually usable). When
+    both are given explicitly, they are trusted even if the detected device count
+    looks insufficient.
+
+    Pinning is independent of ``concurrent_enabled`` — the two answer different
+    questions. An operator running *several* eval-server processes on one multi-GPU
+    box (one per ``EVAL_SERVER_URLS`` entry) wants each process's king and challenger
+    loaded onto that process's own assigned device pair, so VRAM spreads across GPUs
+    instead of every process piling onto the default device — even if that particular
+    process still runs its own two generations sequentially. So an explicit pair of
+    overrides is honored for *where to load* regardless of ``concurrent_enabled``;
+    only the auto-assigned default (``cuda:0``/``cuda:1`` with no override) requires
+    concurrency to be requested, since picking devices nobody asked for is only
+    justified by the throughput it buys.
+    """
+    both_overridden = bool(king_device_override and challenger_device_override)
+
+    if not concurrent_enabled:
+        if not both_overridden:
+            return DuelDevices(
+                None, None, False,
+                "concurrent generation disabled; king and challenger share the default device",
+            )
+        if king_device_override == challenger_device_override:
+            return DuelDevices(
+                king_device_override, challenger_device_override, False,
+                f"both duelists pinned to {king_device_override!r}; loading there, generating "
+                "sequentially (concurrent generation not requested)",
+            )
+        return DuelDevices(
+            king_device_override, challenger_device_override, False,
+            f"king pinned to {king_device_override}, challenger to {challenger_device_override}; "
+            "loading there, generating sequentially (concurrent generation not requested)",
+        )
+
+    if cuda_device_count < 2 and not both_overridden:
+        return DuelDevices(
+            king_device_override, challenger_device_override, False,
+            f"concurrent generation requested but only {cuda_device_count} CUDA device(s) "
+            "visible (need >=2, or set both LEOMA_KING_DEVICE and LEOMA_CHALLENGER_DEVICE) "
+            "— falling back to sequential single-device generation",
+        )
+
+    king = king_device_override or "cuda:0"
+    challenger = challenger_device_override or "cuda:1"
+    if king == challenger:
+        return DuelDevices(
+            king, challenger, False,
+            f"king and challenger both resolved to {king!r} — concurrent generation would "
+            "not overlap anything; falling back to sequential",
+        )
+    return DuelDevices(
+        king, challenger, True,
+        f"generating concurrently: king on {king}, challenger on {challenger}",
+    )
+
+
+__all__ = ["DuelDevices", "resolve_duel_devices"]

leoma/eval/errors.py

@@ -19,6 +19,36 @@
 """
 from __future__ import annotations
 
+# ---------------------------------------------------------------------------
+# Fatal-CUDA detection — shared by the eval server (whose self-kill logic must
+# see it) and the duel runner (whose concurrent generation must never let one
+# fatal exception get silently outrun by a benign one from the other thread).
+# ---------------------------------------------------------------------------
+#
+# Once a CUDA context is corrupted — an illegal memory access, a device-side
+# assert, a cuBLAS execution failure — the VRAM allocator and every stream on
+# this process are poisoned, and every subsequent call keeps raising against the
+# same dead context. This module is the one place both sides check for it, so
+# the token list can't drift into two different definitions of "fatal."
+_CUDA_FATAL_TOKENS = (
+    "an illegal memory access",
+    "cudaerrorillegaladdress",
+    "device-side assert",
+    "cuda error: misaligned address",
+    "cuda error: unspecified launch failure",
+    "cuda error: an illegal instruction",
+    "cublas_status_execution_failed",
+    "cublas_status_not_initialized",
+    "cudnn_status_execution_failed",
+    "bus error",
+    "segmentation fault",
+)
+
+
+def is_cuda_fatal(exc_or_msg) -> bool:
+    """Does this error mean the CUDA context is unrecoverable (not just this duel)?"""
+    return any(tok in str(exc_or_msg or "").lower() for tok in _CUDA_FATAL_TOKENS)
+
 
 class DuelError(RuntimeError):
     """Base class for every duel-path failure."""
@@ -92,4 +122,5 @@ class DuelCancelled(TransientDuelError):
     "ChallengerFault",
     "DegenerateGeneration",
     "DuelCancelled",
+    "is_cuda_fatal",
 ]