From 3690b52f3c1c40c8c7ab5a3095090d8bd460f17b Mon Sep 17 00:00:00 2001
From: vex <223968222+vex0209-bt@users.noreply.github.com>
Date: Tue, 14 Jul 2026 13:12:32 +0000
Subject: [PATCH 1/2] feat: generate king and challenger concurrently on
 separate devices
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

King and challenger generate sequentially today, even on an 8xH100 box where seven
GPUs sit idle for the length of every duel. Clips are independent and use the same
per-clip seed, so the two duelists can generate at the same time on separate devices,
roughly halving duel wall-clock.

This is a throughput setting, not a consensus one: it is resolved from the eval box's
own environment, never sent by the validator or hashed into consensus_digest, and
defaults off (today's sequential single-device behavior is unchanged unless
LEOMA_CONCURRENT_GENERATION=1 is set, and falls back automatically with fewer than 2
CUDA devices). The device each duelist ran on is recorded in the verdict's audit block
so a distance disagreement can be checked against a device asymmetry before assuming a
real bug.

Operator note: enabling this changes what noise delta_threshold has to absorb — even
two identical H100s are not provably bit-identical generators. Re-run `leoma calibrate`
with it enabled before relying on it in production.
---
 env.eval.example                         |  12 ++
 leoma/eval/devices.py                    |  95 ++++++++++++
 leoma/eval/video_runner.py               | 146 ++++++++++++------
 leoma/eval_server.py                     |  46 +++++-
 tests/unit/test_concurrent_generation.py | 187 +++++++++++++++++++++++
 tests/unit/test_devices.py               | 103 +++++++++++++
 6 files changed, 538 insertions(+), 51 deletions(-)
 create mode 100644 leoma/eval/devices.py
 create mode 100644 tests/unit/test_concurrent_generation.py
 create mode 100644 tests/unit/test_devices.py

diff --git a/env.eval.example b/env.eval.example
index cc67ec6..fabe5b0 100644
--- a/env.eval.example
+++ b/env.eval.example
@@ -17,5 +17,17 @@ LEOMA_MODEL_CACHE_DIR=/tmp/leoma/hippius_models
 R2_VIDEOS_READ_ACCESS_KEY=
 R2_VIDEOS_READ_SECRET_KEY=
 
+# Concurrent king/challenger generation — a THROUGHPUT setting, not a consensus one
+# (it is never sent by the validator and never hashed into consensus_digest). On a
+# multi-GPU box, generating both duelists at the same time on separate devices
+# roughly halves duel wall-clock. Falls back to today's sequential single-device
+# behavior if fewer than 2 CUDA devices are visible. See leoma/eval/devices.py.
+#
+# ⚠️ If you enable this, re-run `leoma calibrate` with it enabled — it changes what
+# noise delta_threshold has to absorb (see docs/TESTNET_RUNBOOK.md).
+# LEOMA_CONCURRENT_GENERATION=1
+# LEOMA_KING_DEVICE=cuda:0
+# LEOMA_CHALLENGER_DEVICE=cuda:1
+
 # Optional deterministic prompt for the I2V pipeline (same for king + challenger).
 # LEOMA_DUEL_PROMPT=
diff --git a/leoma/eval/devices.py b/leoma/eval/devices.py
new file mode 100644
index 0000000..94800af
--- /dev/null
+++ b/leoma/eval/devices.py
@@ -0,0 +1,95 @@
+"""Which physical GPU each duelist generates on — a throughput knob, not a consensus one.
+
+Today king and challenger load onto the **same** default CUDA device and generate
+**sequentially**: king's clip, then the challenger's clip, one at a time. On an 8×H100
+box that wastes seven idle GPUs for the length of every duel. Since each clip is
+independent and both duelists use the same per-clip seed, king and challenger can
+generate on two *separate* devices at the same time — roughly halving duel wall-clock.
+
+**Why this is deliberately NOT part of the consensus surface** (``chain.toml`` /
+:class:`~leoma.eval.spec.ConsensusSpec`): generation was already conceded to be
+non-bit-exact across GPU *architectures* (see ``determinism.py`` and
+``eval/calibrate.py``), and ``delta_threshold`` exists precisely to absorb that noise
+regardless of its source. Whether two duelists happen to run on the same physical
+card or two different (but identical) cards in the same box is exactly that kind of
+generation-side noise, not a fact about the exam. So it is a per-box performance
+setting, resolved from the environment, exactly like ``metric_device``.
+
+**The one thing operators must not forget:** enabling this changes *what* noise
+``delta_threshold`` has to absorb. Even two identical H100s in one node are not
+provably bit-identical generators. If you turn this on, re-run
+``leoma calibrate`` with it enabled — you are calibrating against the runtime
+configuration you intend to use in production, not the one you happened to measure
+before you turned this on. See ``docs/TESTNET_RUNBOOK.md``.
+
+This module is a pure function of already-known inputs (feature flag, device count,
+operator overrides) so the *decision* is unit-testable with no GPU. The one line that
+needs an actual CUDA runtime — ``torch.cuda.device_count()`` — lives in the eval
+server's tiny, untested glue, not here.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class DuelDevices:
+    """Where each duelist should load. ``None`` means "today's default device"."""
+
+    king_device: Optional[str]
+    challenger_device: Optional[str]
+    #: True only when generation will ACTUALLY overlap. False whenever the request
+    #: for concurrency couldn't be honored (fewer than 2 devices, or a collision) —
+    #: callers branch on this, not on whether concurrency was merely *requested*.
+    concurrent: bool
+    #: Human-readable explanation of the decision, for logs and the verdict's audit block.
+    note: str
+
+
+def resolve_duel_devices(
+    *,
+    concurrent_enabled: bool,
+    cuda_device_count: int,
+    king_device_override: Optional[str] = None,
+    challenger_device_override: Optional[str] = None,
+) -> DuelDevices:
+    """Decide where king and challenger generate. Falls back to today's behavior
+    whenever concurrency can't be honored safely, rather than guessing.
+
+    ``king_device_override`` / ``challenger_device_override`` let an operator pin
+    specific devices (e.g. to dodge a card another process is using, or on a box
+    where ``torch.cuda.device_count()`` undercounts what's actually usable). When
+    both are given explicitly, they are trusted even if the detected device count
+    looks insufficient.
+    """
+    if not concurrent_enabled:
+        return DuelDevices(
+            None, None, False,
+            "concurrent generation disabled; king and challenger share the default device",
+        )
+
+    both_overridden = bool(king_device_override and challenger_device_override)
+    if cuda_device_count < 2 and not both_overridden:
+        return DuelDevices(
+            king_device_override, challenger_device_override, False,
+            f"concurrent generation requested but only {cuda_device_count} CUDA device(s) "
+            "visible (need >=2, or set both LEOMA_KING_DEVICE and LEOMA_CHALLENGER_DEVICE) "
+            "— falling back to sequential single-device generation",
+        )
+
+    king = king_device_override or "cuda:0"
+    challenger = challenger_device_override or "cuda:1"
+    if king == challenger:
+        return DuelDevices(
+            king, challenger, False,
+            f"king and challenger both resolved to {king!r} — concurrent generation would "
+            "not overlap anything; falling back to sequential",
+        )
+    return DuelDevices(
+        king, challenger, True,
+        f"generating concurrently: king on {king}, challenger on {challenger}",
+    )
+
+
+__all__ = ["DuelDevices", "resolve_duel_devices"]
diff --git a/leoma/eval/video_runner.py b/leoma/eval/video_runner.py
index b49a81d..4f0ec20 100644
--- a/leoma/eval/video_runner.py
+++ b/leoma/eval/video_runner.py
@@ -18,9 +18,17 @@
   digests say immediately whether the generations differed (GPU noise) or only the
   scoring did (a broken box);
 * **the clip's manifest id**, alongside the index the seed was derived from.
+
+``run_duel`` can also generate king and challenger **concurrently** (``concurrent=
+True``): each clip's two generations are independent (same seed, different pipeline),
+so on a multi-GPU box they can run on separate devices at the same time instead of
+one after the other. This is a throughput setting, not a consensus one — see
+``eval/devices.py`` for why, and for the one thing an operator must not forget when
+turning it on.
 """
 from __future__ import annotations
 
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from typing import Callable, Optional, Sequence
 
@@ -85,6 +93,29 @@ class Clip:
 GenerateFn = Callable[[Clip, int], np.ndarray]
 
 
+def _generate_pair(
+    generate_king: GenerateFn,
+    generate_challenger: GenerateFn,
+    clip: Clip,
+    gseed: int,
+    executor: Optional[ThreadPoolExecutor],
+) -> tuple[np.ndarray, np.ndarray]:
+    """Run both generators for one clip — concurrently if an executor is given.
+
+    Only the raw pipeline call is concurrent; validation, digesting and scoring stay
+    on the caller's thread afterward exactly as in the sequential path, so this is the
+    *only* place threading touches the duel at all.
+    """
+    if executor is None:
+        return generate_king(clip, gseed), generate_challenger(clip, gseed)
+    king_future = executor.submit(generate_king, clip, gseed)
+    chall_future = executor.submit(generate_challenger, clip, gseed)
+    # If both raise, the king's exception surfaces first — matching the sequential
+    # path, where the king generates before the challenger and a king-side failure
+    # would be seen first.
+    return king_future.result(), chall_future.result()
+
+
 def run_duel(
     clips: Sequence[Clip],
     generate_king: GenerateFn,
@@ -99,6 +130,7 @@ def run_duel(
     early_stop_max_advantage: Optional[float] = None,
     should_cancel: Optional[Callable[[], bool]] = None,
     freeze_margin_fraction: Optional[float] = None,
+    concurrent: bool = False,
 ) -> dict:
     """Score every clip and return the verdict plus per-clip distances.
 
@@ -114,6 +146,13 @@ def run_duel(
 
     ``freeze_margin_fraction`` enables the freeze-baseline gate: the challenger must
     also beat a frozen conditioning frame, confidently. See ``eval/baselines.py``.
+
+    ``concurrent=True`` runs each clip's king and challenger generation at the same
+    time (on separate devices, if the pipelines were loaded onto separate devices —
+    see ``eval/devices.py``). It changes only *when* the two generations happen, never
+    *what* they compute: the seed, the pipeline, and every downstream step (validation,
+    digesting, scoring) are identical either way. A pinned test asserts the two modes
+    produce byte-identical verdicts.
     """
     if not clips:
         raise ValueError("no clips to duel on")
@@ -122,60 +161,65 @@ def run_duel(
     challenger_scores: list[float] = []
     per_clip: list[dict] = []
 
-    for pos, clip in enumerate(clips):
-        if should_cancel and should_cancel():
-            raise DuelCancelled(f"duel cancelled after {pos} of {len(clips)} clips")
+    executor = ThreadPoolExecutor(max_workers=2) if concurrent else None
+    try:
+        for pos, clip in enumerate(clips):
+            if should_cancel and should_cancel():
+                raise DuelCancelled(f"duel cancelled after {pos} of {len(clips)} clips")
 
-        gseed = clip_generation_seed(master_seed, clip.clip_index)
-        p = clip.params
-        expected = int(np.asarray(clip.truth_frames).shape[0])
+            gseed = clip_generation_seed(master_seed, clip.clip_index)
+            p = clip.params
+            expected = int(np.asarray(clip.truth_frames).shape[0])
 
-        king_frames = validate_generation(
-            generate_king(clip, gseed),
-            expected_frames=expected, width=p.width, height=p.height, who="king",
-        )
-        chall_frames = validate_generation(
-            generate_challenger(clip, gseed),
-            expected_frames=expected, width=p.width, height=p.height, who="challenger",
-        )
+            raw_king, raw_chall = _generate_pair(generate_king, generate_challenger, clip, gseed, executor)
 
-        kd = float(distance_fn(king_frames, clip.truth_frames))
-        cd = float(distance_fn(chall_frames, clip.truth_frames))
-        king_scores.append(kd)
-        challenger_scores.append(cd)
-        per_clip.append({
-            "clip_index": clip.clip_index,
-            "clip_id": clip.clip_id,
-            "gen_seed": gseed,
-            "king_distance": kd,
-            "challenger_distance": cd,
-            "king_frames_digest": digest_frames(king_frames),
-            "challenger_frames_digest": digest_frames(chall_frames),
-        })
+            king_frames = validate_generation(
+                raw_king, expected_frames=expected, width=p.width, height=p.height, who="king",
+            )
+            chall_frames = validate_generation(
+                raw_chall, expected_frames=expected, width=p.width, height=p.height, who="challenger",
+            )
 
-        if on_phase:
-            on_phase({
-                "phase": "scored_clip",
-                "position": pos + 1,
-                "total": len(clips),
+            kd = float(distance_fn(king_frames, clip.truth_frames))
+            cd = float(distance_fn(chall_frames, clip.truth_frames))
+            king_scores.append(kd)
+            challenger_scores.append(cd)
+            per_clip.append({
+                "clip_index": clip.clip_index,
                 "clip_id": clip.clip_id,
-                "king_distance": round(kd, 6),
-                "challenger_distance": round(cd, 6),
+                "gen_seed": gseed,
+                "king_distance": kd,
+                "challenger_distance": cd,
+                "king_frames_digest": digest_frames(king_frames),
+                "challenger_frames_digest": digest_frames(chall_frames),
             })
 
-        if early_stop_max_advantage is not None and not can_still_win(
-            king_scores, challenger_scores, remaining=len(clips) - (pos + 1),
-            delta_threshold=delta_threshold, best_possible_advantage=early_stop_max_advantage,
-        ):
             if on_phase:
-                on_phase({"phase": "early_stop", "reason": "challenger cannot clear threshold"})
-            verdict = paired_bootstrap_verdict(
-                king_scores, challenger_scores,
-                delta_threshold=delta_threshold, alpha=alpha, n_bootstrap=n_bootstrap, seed=master_seed,
-            )
-            verdict["early_stopped"] = True
-            verdict["per_clip"] = per_clip
-            return verdict
+                on_phase({
+                    "phase": "scored_clip",
+                    "position": pos + 1,
+                    "total": len(clips),
+                    "clip_id": clip.clip_id,
+                    "king_distance": round(kd, 6),
+                    "challenger_distance": round(cd, 6),
+                })
+
+            if early_stop_max_advantage is not None and not can_still_win(
+                king_scores, challenger_scores, remaining=len(clips) - (pos + 1),
+                delta_threshold=delta_threshold, best_possible_advantage=early_stop_max_advantage,
+            ):
+                if on_phase:
+                    on_phase({"phase": "early_stop", "reason": "challenger cannot clear threshold"})
+                verdict = paired_bootstrap_verdict(
+                    king_scores, challenger_scores,
+                    delta_threshold=delta_threshold, alpha=alpha, n_bootstrap=n_bootstrap, seed=master_seed,
+                )
+                verdict["early_stopped"] = True
+                verdict["per_clip"] = per_clip
+                return verdict
+    finally:
+        if executor is not None:
+            executor.shutdown(wait=True)
 
     verdict = paired_bootstrap_verdict(
         king_scores, challenger_scores,
@@ -424,8 +468,15 @@ def duel(
     early_stop_max_advantage: Optional[float] = None,
     should_cancel: Optional[Callable[[], bool]] = None,
     freeze_margin_fraction: Optional[float] = None,
+    concurrent: bool = False,
 ) -> dict:
-    """Production wrapper: bind loaded pipelines + the named metric into ``run_duel``."""
+    """Production wrapper: bind loaded pipelines + the named metric into ``run_duel``.
+
+    ``concurrent=True`` only helps if ``king_pipeline`` and ``challenger_pipeline``
+    were loaded onto *different* devices (``eval/devices.py``) — generating
+    concurrently on the same device just serializes on that device's queue anyway,
+    so the caller is responsible for both being true together.
+    """
     gen = generate_fn or generate
     distance_fn = get_metric(metric, device=metric_device)
     return run_duel(
@@ -441,4 +492,5 @@ def duel(
         early_stop_max_advantage=early_stop_max_advantage,
         should_cancel=should_cancel,
         freeze_margin_fraction=freeze_margin_fraction,
+        concurrent=concurrent,
     )
diff --git a/leoma/eval_server.py b/leoma/eval_server.py
index 990eba5..f40459a 100644
--- a/leoma/eval_server.py
+++ b/leoma/eval_server.py
@@ -503,6 +503,27 @@ def check_request(req: EvalRequest) -> None:
     req.spec.require_duel_ready()
 
 
+def _resolve_devices():
+    """Glue: read the concurrency env vars + the real CUDA device count.
+
+    The decision itself (``leoma.eval.devices.resolve_duel_devices``) is pure and
+    unit-tested without a GPU; this function exists only because *something* has to
+    call ``torch.cuda.device_count()``, and that something needs torch installed.
+    """
+    import os
+
+    import torch
+
+    from leoma.eval.devices import resolve_duel_devices
+
+    return resolve_duel_devices(
+        concurrent_enabled=os.environ.get("LEOMA_CONCURRENT_GENERATION", "0") == "1",
+        cuda_device_count=torch.cuda.device_count() if torch.cuda.is_available() else 0,
+        king_device_override=os.environ.get("LEOMA_KING_DEVICE") or None,
+        challenger_device_override=os.environ.get("LEOMA_CHALLENGER_DEVICE") or None,
+    )
+
+
 def _download_watcher(emit: Emit, which: str, path, stop: threading.Event) -> None:
     """Report a download's forward progress so the watchdog can tell slow from hung.
 
@@ -569,14 +590,22 @@ def _materialize(ref, which: str) -> str:
     king_dir = _materialize(king_ref, "king")
     chall_dir = _materialize(chall_ref, "challenger")
 
+    # A throughput setting, not a consensus one — see eval/devices.py. Resolved from
+    # this box's own environment, never from the request: if it changed per-request
+    # the cached king pipeline (below) could silently end up on the wrong device.
+    devices = _resolve_devices()
+    emit({"phase": "devices", "concurrent": devices.concurrent, "note": devices.note})
+
     chall_pipe = None
     try:
-        emit({"phase": "load", "which": "king"})
+        emit({"phase": "load", "which": "king", "device": devices.king_device})
         # Keyed by the king's immutable ref: the same king faces every challenger in
         # the queue, and re-loading 28 GB into VRAM for each one is pure waste.
-        king_pipe = load_video_pipeline(king_dir, gen=spec.gen, cache_key=king_ref.immutable_ref)
-        emit({"phase": "load", "which": "challenger"})
-        chall_pipe = load_video_pipeline(chall_dir, gen=spec.gen)
+        king_pipe = load_video_pipeline(
+            king_dir, gen=spec.gen, device=devices.king_device, cache_key=king_ref.immutable_ref
+        )
+        emit({"phase": "load", "which": "challenger", "device": devices.challenger_device})
+        chall_pipe = load_video_pipeline(chall_dir, gen=spec.gen, device=devices.challenger_device)
 
         master_seed = eval_seed(req.block_hash, req.hotkey, spec.duel.base_seed)
         params = GenParams.from_spec(spec.gen)
@@ -611,6 +640,7 @@ def _materialize(ref, which: str) -> str:
             early_stop_max_advantage=spec.early_stop_max_advantage,
             should_cancel=should_cancel,
             freeze_margin_fraction=spec.duel.freeze_margin_fraction,
+            concurrent=devices.concurrent,
         )
     finally:
         # The challenger is done with either way — success, error, or cancellation.
@@ -636,6 +666,14 @@ def _materialize(ref, which: str) -> str:
         "eval_code_digest": eval_code_digest(),
         "corpus": corpus_audit(manifest, entries),
         "env": runtime_env(),
+        # Not part of the consensus surface (see eval/devices.py) — recorded here so a
+        # distance disagreement between validators can be checked against "did the two
+        # duelists even run on the same kind of device" before assuming a real bug.
+        "generation": {
+            "concurrent": devices.concurrent,
+            "king_device": devices.king_device,
+            "challenger_device": devices.challenger_device,
+        },
     }
     # Hashes ONLY the consensus surface — the decision, the exam, the parameters.
     # Deliberately excludes env and produced_at: two validators on different GPUs
diff --git a/tests/unit/test_concurrent_generation.py b/tests/unit/test_concurrent_generation.py
new file mode 100644
index 0000000..d72ae7c
--- /dev/null
+++ b/tests/unit/test_concurrent_generation.py
@@ -0,0 +1,187 @@
+"""Concurrent king/challenger generation.
+
+The only thing ``concurrent=True`` may change is *when* the two generations happen,
+never *what* they compute. The single most important test in this file is
+``test_concurrent_and_sequential_produce_byte_identical_verdicts`` — if that one ever
+fails, the throughput change has become a consensus bug.
+"""
+
+import threading
+import time
+
+import numpy as np
+import pytest
+
+from leoma.eval.metrics import mse
+from leoma.eval.video_runner import Clip, GenParams, run_duel
+
+PARAMS = GenParams(num_frames=4, fps=2, width=8, height=8)
+SLEEP = 0.06  # generous relative to scheduling jitter, small enough for a fast test
+
+
+def _clips(n=6, seed=0):
+    rng = np.random.default_rng(seed)
+    clips = []
+    for i in range(n):
+        truth = rng.integers(0, 255, size=(4, 8, 8, 3)).astype("uint8")
+        clips.append(Clip(clip_index=i, clip_id=f"clip-{i:04d}", first_frame=truth[0],
+                          prompt="p", truth_frames=truth, params=PARAMS))
+    return clips
+
+
+def _slow_generate(offset: int):
+    """A deterministic generator that also takes real wall-clock time, so the two
+    duelists' generations can genuinely overlap in a thread pool."""
+    def gen(clip: Clip, seed: int) -> np.ndarray:
+        time.sleep(SLEEP)
+        return np.clip(clip.truth_frames.astype(int) + offset + (seed % 7), 0, 255).astype("uint8")
+    return gen
+
+
+def _run(clips, concurrent, seed=42):
+    return run_duel(
+        clips, generate_king=_slow_generate(3), generate_challenger=_slow_generate(5),
+        distance_fn=mse, master_seed=seed, delta_threshold=0.0025, alpha=0.05,
+        n_bootstrap=200, concurrent=concurrent,
+    )
+
+
+class TestByteIdenticalRegardlessOfConcurrency:
+    """THE property this feature must never violate."""
+
+    def test_concurrent_and_sequential_produce_byte_identical_verdicts(self):
+        clips = _clips()
+        sequential = _run(clips, concurrent=False)
+        concurrent = _run(clips, concurrent=True)
+
+        assert sequential["accepted"] == concurrent["accepted"]
+        assert sequential["mu_hat"] == concurrent["mu_hat"]
+        assert sequential["lcb"] == concurrent["lcb"]
+        assert sequential["avg_king_distance"] == concurrent["avg_king_distance"]
+        assert sequential["avg_challenger_distance"] == concurrent["avg_challenger_distance"]
+
+        for a, b in zip(sequential["per_clip"], concurrent["per_clip"]):
+            assert a["king_distance"] == b["king_distance"]
+            assert a["challenger_distance"] == b["challenger_distance"]
+            assert a["king_frames_digest"] == b["king_frames_digest"]
+            assert a["challenger_frames_digest"] == b["challenger_frames_digest"]
+            assert a["gen_seed"] == b["gen_seed"]
+
+    def test_the_per_clip_seed_pairing_is_unaffected_by_concurrency(self):
+        seen = {"king": [], "chall": []}
+
+        def gk(clip, seed):
+            seen["king"].append((clip.clip_index, seed))
+            return np.clip(clip.truth_frames.astype(int) + 1, 0, 255).astype("uint8")
+
+        def gc(clip, seed):
+            seen["chall"].append((clip.clip_index, seed))
+            return np.clip(clip.truth_frames.astype(int) + 2, 0, 255).astype("uint8")
+
+        run_duel(_clips(n=4), gk, gc, mse, master_seed=7, delta_threshold=0.0025,
+                 alpha=0.05, n_bootstrap=100, concurrent=True)
+        assert seen["king"] == seen["chall"]
+
+
+class TestActualOverlap:
+    def test_concurrent_is_meaningfully_faster_than_sequential(self):
+        """time.sleep() releases the GIL, so two threads sleeping in parallel genuinely
+        overlap — this is a reliable way to prove concurrency without a real GPU."""
+        clips = _clips(n=8)
+
+        t0 = time.monotonic()
+        _run(clips, concurrent=False)
+        sequential_elapsed = time.monotonic() - t0
+
+        t0 = time.monotonic()
+        _run(clips, concurrent=True)
+        concurrent_elapsed = time.monotonic() - t0
+
+        # Sequential: ~2 * n * SLEEP. Concurrent: ~1 * n * SLEEP. A 1.5x margin is
+        # comfortably below the theoretical 2x speedup, avoiding CI flakiness.
+        assert concurrent_elapsed < sequential_elapsed / 1.5, (
+            f"concurrent ({concurrent_elapsed:.3f}s) was not meaningfully faster than "
+            f"sequential ({sequential_elapsed:.3f}s)"
+        )
+
+
+class TestFailureModes:
+    def test_a_king_side_exception_propagates(self):
+        def boom(clip, seed):
+            raise RuntimeError("king pipeline OOM")
+
+        with pytest.raises(RuntimeError, match="king pipeline OOM"):
+            run_duel(_clips(n=2), boom, _slow_generate(0), mse, master_seed=1,
+                     delta_threshold=0.0025, alpha=0.05, n_bootstrap=50, concurrent=True)
+
+    def test_a_challenger_side_exception_propagates(self):
+        def boom(clip, seed):
+            raise RuntimeError("challenger pipeline OOM")
+
+        with pytest.raises(RuntimeError, match="challenger pipeline OOM"):
+            run_duel(_clips(n=2), _slow_generate(0), boom, mse, master_seed=1,
+                     delta_threshold=0.0025, alpha=0.05, n_bootstrap=50, concurrent=True)
+
+    def test_cancellation_between_clips_still_works_when_concurrent(self):
+        from leoma.eval.errors import DuelCancelled
+
+        calls = {"n": 0}
+
+        def cancel_after_one():
+            calls["n"] += 1
+            return calls["n"] > 1
+
+        with pytest.raises(DuelCancelled):
+            run_duel(_clips(n=5), _slow_generate(0), _slow_generate(1), mse, master_seed=1,
+                     delta_threshold=0.0025, alpha=0.05, n_bootstrap=50,
+                     concurrent=True, should_cancel=cancel_after_one)
+
+    def test_a_degenerate_challenger_is_still_caught_after_concurrent_generation(self):
+        from leoma.eval.errors import DegenerateGeneration
+
+        def one_frame(clip, seed):
+            return np.zeros((1, 8, 8, 3), dtype="uint8")
+
+        with pytest.raises(DegenerateGeneration):
+            run_duel(_clips(n=2), _slow_generate(0), one_frame, mse, master_seed=1,
+                     delta_threshold=0.0025, alpha=0.05, n_bootstrap=50, concurrent=True)
+
+
+class TestNoThreadLeak:
+    def test_the_executor_is_torn_down_after_every_run(self):
+        baseline = threading.active_count()
+        for _ in range(5):
+            _run(_clips(n=2), concurrent=True)
+        # Give any lingering worker threads a moment to actually exit.
+        for _ in range(20):
+            if threading.active_count() <= baseline:
+                break
+            time.sleep(0.02)
+        assert threading.active_count() <= baseline
+
+    def test_torn_down_even_when_the_duel_raises(self):
+        baseline = threading.active_count()
+
+        def boom(clip, seed):
+            raise RuntimeError("x")
+
+        for _ in range(5):
+            with pytest.raises(RuntimeError):
+                run_duel(_clips(n=2), boom, _slow_generate(0), mse, master_seed=1,
+                         delta_threshold=0.0025, alpha=0.05, n_bootstrap=50, concurrent=True)
+
+        for _ in range(20):
+            if threading.active_count() <= baseline:
+                break
+            time.sleep(0.02)
+        assert threading.active_count() <= baseline
+
+
+class TestSequentialIsUnaffected:
+    """concurrent defaults to False — every existing caller's behavior is untouched."""
+
+    def test_default_is_sequential(self):
+        clips = _clips(n=2)
+        v = run_duel(clips, _slow_generate(0), _slow_generate(1), mse, master_seed=1,
+                     delta_threshold=0.0025, alpha=0.05, n_bootstrap=50)
+        assert v["per_clip"][0]["king_distance"] is not None  # ran to completion normally
diff --git a/tests/unit/test_devices.py b/tests/unit/test_devices.py
new file mode 100644
index 0000000..f074b21
--- /dev/null
+++ b/tests/unit/test_devices.py
@@ -0,0 +1,103 @@
+"""Device assignment for concurrent king/challenger generation.
+
+A throughput setting, not a consensus one — see eval/devices.py's module docstring for
+why. These tests pin the decision logic: when concurrency is actually honored, and the
+several ways it safely falls back to today's single-device behavior instead of
+guessing.
+"""
+
+import pytest
+
+from leoma.eval.devices import resolve_duel_devices
+
+
+class TestDisabled:
+    def test_disabled_means_no_devices_assigned(self):
+        d = resolve_duel_devices(concurrent_enabled=False, cuda_device_count=8)
+        assert d.king_device is None
+        assert d.challenger_device is None
+        assert d.concurrent is False
+
+
+class TestInsufficientDevices:
+    @pytest.mark.parametrize("count", [0, 1])
+    def test_fewer_than_two_devices_falls_back(self, count):
+        d = resolve_duel_devices(concurrent_enabled=True, cuda_device_count=count)
+        assert d.concurrent is False
+        assert "falling back" in d.note
+
+    def test_the_fallback_explains_itself(self):
+        d = resolve_duel_devices(concurrent_enabled=True, cuda_device_count=1)
+        assert "1 CUDA device" in d.note
+
+
+class TestHappyPath:
+    def test_two_devices_assigns_king_and_challenger_separately(self):
+        d = resolve_duel_devices(concurrent_enabled=True, cuda_device_count=2)
+        assert d.concurrent is True
+        assert d.king_device == "cuda:0"
+        assert d.challenger_device == "cuda:1"
+
+    def test_more_than_two_devices_still_uses_the_first_two(self):
+        d = resolve_duel_devices(concurrent_enabled=True, cuda_device_count=8)
+        assert d.king_device == "cuda:0"
+        assert d.challenger_device == "cuda:1"
+        assert d.concurrent is True
+
+
+class TestOverrides:
+    def test_explicit_overrides_are_respected(self):
+        d = resolve_duel_devices(
+            concurrent_enabled=True, cuda_device_count=8,
+            king_device_override="cuda:2", challenger_device_override="cuda:3",
+        )
+        assert d.king_device == "cuda:2"
+        assert d.challenger_device == "cuda:3"
+        assert d.concurrent is True
+
+    def test_both_overrides_are_trusted_even_with_a_low_reported_count(self):
+        """torch.cuda.device_count() might undercount what an operator knows is usable
+        (MIG partitions, CUDA_VISIBLE_DEVICES tricks) — an explicit pair of overrides
+        is trusted rather than second-guessed."""
+        d = resolve_duel_devices(
+            concurrent_enabled=True, cuda_device_count=1,
+            king_device_override="cuda:0", challenger_device_override="cuda:1",
+        )
+        assert d.concurrent is True
+
+    def test_a_single_override_with_too_few_devices_still_falls_back(self):
+        """Only ONE side pinned isn't enough information to safely pick the other."""
+        d = resolve_duel_devices(
+            concurrent_enabled=True, cuda_device_count=1, king_device_override="cuda:0",
+        )
+        assert d.concurrent is False
+
+    def test_overrides_colliding_on_the_same_device_falls_back(self):
+        """Generating 'concurrently' on the same device just serializes on that
+        device's queue — no overlap, so it must not be reported as concurrent."""
+        d = resolve_duel_devices(
+            concurrent_enabled=True, cuda_device_count=8,
+            king_device_override="cuda:0", challenger_device_override="cuda:0",
+        )
+        assert d.concurrent is False
+        assert "not overlap" in d.note
+
+    def test_a_default_challenger_colliding_with_an_overridden_king_falls_back(self):
+        """King is pinned to cuda:1 (the challenger's default) with no explicit
+        challenger override — the collision must still be caught."""
+        d = resolve_duel_devices(
+            concurrent_enabled=True, cuda_device_count=8, king_device_override="cuda:1",
+        )
+        assert d.challenger_device == "cuda:1"
+        assert d.concurrent is False
+
+
+class TestNoteIsAlwaysPresent:
+    def test_every_outcome_explains_itself(self):
+        for kwargs in (
+            dict(concurrent_enabled=False, cuda_device_count=8),
+            dict(concurrent_enabled=True, cuda_device_count=0),
+            dict(concurrent_enabled=True, cuda_device_count=2),
+        ):
+            d = resolve_duel_devices(**kwargs)
+            assert d.note and isinstance(d.note, str)

From e2dd99ccfbd3c6753fdf2df4f2275c210b9aca2b Mon Sep 17 00:00:00 2001
From: vex <223968222+vex0209-bt@users.noreply.github.com>
Date: Tue, 14 Jul 2026 14:35:16 +0000
Subject: [PATCH 2/2] fix: never drop either generation's exception, and
 decouple device pinning from concurrency
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review found two real bugs before merge. First: _generate_pair called
king_future.result() before chall_future.result(), so if the king's generation
raised a benign error the challenger's exception was never retrieved at all — an
exception nobody calls .result()/.exception() on is simply gone, not logged, not
re-raised. If that dropped exception was CUDA-fatal, the eval server's self-kill
check never saw it, released the lock, and handed the next challenger a poisoned
GPU context. Both futures are now always retrieved, and whichever side is
CUDA-fatal is what propagates, regardless of which one happened to be checked
first. is_cuda_fatal moves to eval/errors.py so both the eval server and the duel
runner check the same token list.

Second: resolve_duel_devices discarded explicit device overrides whenever
concurrent generation was disabled, but pinning ("which devices to load onto") and
concurrency ("do the two generations overlap in time") are different questions —
an operator running several eval-server processes on one multi-GPU box wants each
process's king+challenger spread across its own device pair to avoid piling every
process onto the same default GPU, even if that process still generates
sequentially. Overrides are now honored for loading regardless of the concurrency
flag; only the auto-assigned cuda:0/cuda:1 default still requires concurrency to
be requested.
---
 env.eval.example                         |   6 ++
 leoma/eval/devices.py                    |  30 ++++++-
 leoma/eval/errors.py                     |  31 +++++++
 leoma/eval/video_runner.py               |  35 ++++++--
 leoma/eval_server.py                     |  25 ++----
 tests/unit/test_concurrent_generation.py | 110 +++++++++++++++++++++++
 tests/unit/test_devices.py               |  48 ++++++++++
 7 files changed, 258 insertions(+), 27 deletions(-)

diff --git a/env.eval.example b/env.eval.example
index fabe5b0..4759002 100644
--- a/env.eval.example
+++ b/env.eval.example
@@ -26,6 +26,12 @@ R2_VIDEOS_READ_SECRET_KEY=
 # ⚠️ If you enable this, re-run `leoma calibrate` with it enabled — it changes what
 # noise delta_threshold has to absorb (see docs/TESTNET_RUNBOOK.md).
 # LEOMA_CONCURRENT_GENERATION=1
+
+# LEOMA_KING_DEVICE / LEOMA_CHALLENGER_DEVICE are independent of the concurrency flag
+# above: they also matter with LEOMA_CONCURRENT_GENERATION unset, e.g. when running
+# several eval-server PROCESSES on one multi-GPU box (one per EVAL_SERVER_URLS entry)
+# and pinning each process to its own device pair so their VRAM doesn't all pile onto
+# the same default GPU. Set both together, or neither — a single one is ignored.
 # LEOMA_KING_DEVICE=cuda:0
 # LEOMA_CHALLENGER_DEVICE=cuda:1
 
diff --git a/leoma/eval/devices.py b/leoma/eval/devices.py
index 94800af..3e25f4b 100644
--- a/leoma/eval/devices.py
+++ b/leoma/eval/devices.py
@@ -62,14 +62,38 @@ def resolve_duel_devices(
     where ``torch.cuda.device_count()`` undercounts what's actually usable). When
     both are given explicitly, they are trusted even if the detected device count
     looks insufficient.
+
+    Pinning is independent of ``concurrent_enabled`` — the two answer different
+    questions. An operator running *several* eval-server processes on one multi-GPU
+    box (one per ``EVAL_SERVER_URLS`` entry) wants each process's king and challenger
+    loaded onto that process's own assigned device pair, so VRAM spreads across GPUs
+    instead of every process piling onto the default device — even if that particular
+    process still runs its own two generations sequentially. So an explicit pair of
+    overrides is honored for *where to load* regardless of ``concurrent_enabled``;
+    only the auto-assigned default (``cuda:0``/``cuda:1`` with no override) requires
+    concurrency to be requested, since picking devices nobody asked for is only
+    justified by the throughput it buys.
     """
+    both_overridden = bool(king_device_override and challenger_device_override)
+
     if not concurrent_enabled:
+        if not both_overridden:
+            return DuelDevices(
+                None, None, False,
+                "concurrent generation disabled; king and challenger share the default device",
+            )
+        if king_device_override == challenger_device_override:
+            return DuelDevices(
+                king_device_override, challenger_device_override, False,
+                f"both duelists pinned to {king_device_override!r}; loading there, generating "
+                "sequentially (concurrent generation not requested)",
+            )
         return DuelDevices(
-            None, None, False,
-            "concurrent generation disabled; king and challenger share the default device",
+            king_device_override, challenger_device_override, False,
+            f"king pinned to {king_device_override}, challenger to {challenger_device_override}; "
+            "loading there, generating sequentially (concurrent generation not requested)",
         )
 
-    both_overridden = bool(king_device_override and challenger_device_override)
     if cuda_device_count < 2 and not both_overridden:
         return DuelDevices(
             king_device_override, challenger_device_override, False,
diff --git a/leoma/eval/errors.py b/leoma/eval/errors.py
index 0a50eab..9a94fae 100644
--- a/leoma/eval/errors.py
+++ b/leoma/eval/errors.py
@@ -19,6 +19,36 @@
 """
 from __future__ import annotations
 
+# ---------------------------------------------------------------------------
+# Fatal-CUDA detection — shared by the eval server (whose self-kill logic must
+# see it) and the duel runner (whose concurrent generation must never let one
+# fatal exception get silently outrun by a benign one from the other thread).
+# ---------------------------------------------------------------------------
+#
+# Once a CUDA context is corrupted — an illegal memory access, a device-side
+# assert, a cuBLAS execution failure — the VRAM allocator and every stream on
+# this process are poisoned, and every subsequent call keeps raising against the
+# same dead context. This module is the one place both sides check for it, so
+# the token list can't drift into two different definitions of "fatal."
+_CUDA_FATAL_TOKENS = (
+    "an illegal memory access",
+    "cudaerrorillegaladdress",
+    "device-side assert",
+    "cuda error: misaligned address",
+    "cuda error: unspecified launch failure",
+    "cuda error: an illegal instruction",
+    "cublas_status_execution_failed",
+    "cublas_status_not_initialized",
+    "cudnn_status_execution_failed",
+    "bus error",
+    "segmentation fault",
+)
+
+
+def is_cuda_fatal(exc_or_msg) -> bool:
+    """Does this error mean the CUDA context is unrecoverable (not just this duel)?"""
+    return any(tok in str(exc_or_msg or "").lower() for tok in _CUDA_FATAL_TOKENS)
+
 
 class DuelError(RuntimeError):
     """Base class for every duel-path failure."""
@@ -92,4 +122,5 @@ class DuelCancelled(TransientDuelError):
     "ChallengerFault",
     "DegenerateGeneration",
     "DuelCancelled",
+    "is_cuda_fatal",
 ]
diff --git a/leoma/eval/video_runner.py b/leoma/eval/video_runner.py
index 4f0ec20..2d6b0b9 100644
--- a/leoma/eval/video_runner.py
+++ b/leoma/eval/video_runner.py
@@ -36,7 +36,7 @@
 
 from leoma.eval.bootstrap import can_still_win, paired_bootstrap_verdict
 from leoma.eval.digests import digest_frames
-from leoma.eval.errors import DuelCancelled
+from leoma.eval.errors import DuelCancelled, is_cuda_fatal
 from leoma.eval.guards import validate_generation
 from leoma.eval.metrics import Metric, get_metric
 from leoma.app.validator.seeds import clip_generation_seed
@@ -105,15 +105,40 @@ def _generate_pair(
     Only the raw pipeline call is concurrent; validation, digesting and scoring stay
     on the caller's thread afterward exactly as in the sequential path, so this is the
     *only* place threading touches the duel at all.
+
+    Both futures are always retrieved before either exception is raised. Calling
+    ``king_future.result()`` first and letting it raise immediately would leave
+    ``chall_future``'s outcome never retrieved — and an exception nobody ever calls
+    ``.result()``/``.exception()`` on is simply dropped, not logged, not re-raised,
+    gone. That is exactly how a CUDA-fatal error on the CHALLENGER side could vanish
+    behind a merely-benign KING-side error: the eval server's self-kill logic only
+    ever inspects the one exception that actually propagates out of ``run_duel``, so a
+    dropped fatal exception means a poisoned CUDA context is never detected, the lock
+    is released, and the *next* challenger inherits a dead GPU.
     """
     if executor is None:
         return generate_king(clip, gseed), generate_challenger(clip, gseed)
     king_future = executor.submit(generate_king, clip, gseed)
     chall_future = executor.submit(generate_challenger, clip, gseed)
-    # If both raise, the king's exception surfaces first — matching the sequential
-    # path, where the king generates before the challenger and a king-side failure
-    # would be seen first.
-    return king_future.result(), chall_future.result()
+
+    king_exc = king_future.exception()   # blocks until done; never raises itself
+    chall_exc = chall_future.exception()
+
+    if king_exc is None and chall_exc is None:
+        return king_future.result(), chall_future.result()
+
+    # At least one side failed. A CUDA-fatal exception must be what propagates,
+    # whichever side it came from — the context is poisoned regardless of which
+    # duelist's generate() call happened to surface it. Otherwise, king-first,
+    # matching the order generation always ran in before concurrency existed.
+    chall_is_the_fatal_one = chall_exc is not None and is_cuda_fatal(chall_exc) and (
+        king_exc is None or not is_cuda_fatal(king_exc)
+    )
+    if chall_is_the_fatal_one:
+        raise chall_exc
+    if king_exc is not None:
+        raise king_exc
+    raise chall_exc
 
 
 def run_duel(
diff --git a/leoma/eval_server.py b/leoma/eval_server.py
index f40459a..dd4740a 100644
--- a/leoma/eval_server.py
+++ b/leoma/eval_server.py
@@ -47,7 +47,7 @@
 from fastapi.responses import JSONResponse, StreamingResponse
 from pydantic import BaseModel, ConfigDict
 
-from leoma.eval.errors import DuelCancelled
+from leoma.eval.errors import DuelCancelled, is_cuda_fatal
 from leoma.eval.spec import ConsensusSpec
 
 
@@ -135,29 +135,16 @@ class EvalRequest(BaseModel):
 # `cuda_fatal` as TRANSIENT and retries against a — now freshly restarted — box), then
 # `os._exit` to skip atexit hooks that would touch the corrupted GPU and hang.
 # Ported from Teutonic's eval_server self-kill, which cites a real 2.5h box degradation.
-_CUDA_FATAL_TOKENS = (
-    "an illegal memory access",
-    "cudaerrorillegaladdress",
-    "device-side assert",
-    "cuda error: misaligned address",
-    "cuda error: unspecified launch failure",
-    "cuda error: an illegal instruction",
-    "cublas_status_execution_failed",
-    "cublas_status_not_initialized",
-    "cudnn_status_execution_failed",
-    "bus error",
-    "segmentation fault",
-)
+#
+# is_cuda_fatal() itself lives in eval/errors.py, not here: with concurrent king/
+# challenger generation, the duel runner ALSO needs it (to decide which of two
+# simultaneous exceptions must be the one that propagates), and it must be the same
+# token list on both sides rather than two definitions that can drift apart.
 CUDA_FATAL_EXIT_DELAY_S = float(os.environ.get("LEOMA_CUDA_FATAL_DELAY", "3"))
 CUDA_FATAL_EXIT_CODE = int(os.environ.get("LEOMA_CUDA_FATAL_EXIT_CODE", "75"))
 _self_kill_scheduled = threading.Event()
 
 
-def is_cuda_fatal(exc_or_msg) -> bool:
-    """Does this error mean the CUDA context is unrecoverable (not just this duel)?"""
-    return any(tok in str(exc_or_msg or "").lower() for tok in _CUDA_FATAL_TOKENS)
-
-
 def schedule_self_kill(reason: str, *, delay_s: Optional[float] = None) -> None:
     """Exit the process because CUDA state is poisoned. Idempotent; delayed.
 
diff --git a/tests/unit/test_concurrent_generation.py b/tests/unit/test_concurrent_generation.py
index d72ae7c..467d3e6 100644
--- a/tests/unit/test_concurrent_generation.py
+++ b/tests/unit/test_concurrent_generation.py
@@ -147,6 +147,116 @@ def one_frame(clip, seed):
                      delta_threshold=0.0025, alpha=0.05, n_bootstrap=50, concurrent=True)
 
 
+class TestNeitherFutureIsSilentlyDropped:
+    """THE bug this class exists to catch. `king_future.result()` raising immediately
+    would leave `chall_future`'s outcome never retrieved — and an exception nobody ever
+    calls .result()/.exception() on is simply gone: not logged, not re-raised, not
+    seen by the eval server's is_cuda_fatal() self-kill check. A CUDA-fatal error on
+    ONE side must propagate even when the OTHER side raises first, or a benign king-
+    side error, is CUDA-fatal error and the poisoned context both go undetected.
+    """
+
+    def test_a_challenger_side_CUDA_fatal_error_propagates_over_a_benign_king_error(self):
+        """The exact scenario the review reproduced: king benign, challenger fatal —
+        without the fix, only the benign one ever surfaces."""
+        def king_benign(clip, seed):
+            raise RuntimeError("benign king-side hiccup")
+
+        def chall_fatal(clip, seed):
+            raise RuntimeError("CUDA error: an illegal memory access was encountered")
+
+        with pytest.raises(RuntimeError, match="illegal memory access"):
+            run_duel(_clips(n=2), king_benign, chall_fatal, mse, master_seed=1,
+                     delta_threshold=0.0025, alpha=0.05, n_bootstrap=50, concurrent=True)
+
+    def test_a_king_side_CUDA_fatal_error_still_propagates_over_a_benign_challenger_error(self):
+        def king_fatal(clip, seed):
+            raise RuntimeError("CUDNN_STATUS_EXECUTION_FAILED")
+
+        def chall_benign(clip, seed):
+            raise RuntimeError("benign challenger-side hiccup")
+
+        with pytest.raises(RuntimeError, match="CUDNN_STATUS_EXECUTION_FAILED"):
+            run_duel(_clips(n=2), king_fatal, chall_benign, mse, master_seed=1,
+                     delta_threshold=0.0025, alpha=0.05, n_bootstrap=50, concurrent=True)
+
+    def test_the_propagated_exception_is_something_is_cuda_fatal_recognizes(self):
+        """Directly proves the eval server's self-kill check would actually fire: the
+        exception that ESCAPES run_duel must itself satisfy is_cuda_fatal()."""
+        from leoma.eval.errors import is_cuda_fatal
+
+        def king_benign(clip, seed):
+            raise RuntimeError("benign")
+
+        def chall_fatal(clip, seed):
+            raise RuntimeError("device-side assert triggered")
+
+        try:
+            run_duel(_clips(n=2), king_benign, chall_fatal, mse, master_seed=1,
+                     delta_threshold=0.0025, alpha=0.05, n_bootstrap=50, concurrent=True)
+            assert False, "expected an exception"
+        except RuntimeError as e:
+            assert is_cuda_fatal(e), "the propagated exception must be the CUDA-fatal one"
+
+    def test_when_neither_is_fatal_king_still_wins_matching_the_old_default_order(self):
+        """No behavior change for the common case: both benign -> king's error
+        surfaces, exactly as before concurrency existed."""
+        def king_benign(clip, seed):
+            raise RuntimeError("king benign")
+
+        def chall_benign(clip, seed):
+            raise RuntimeError("challenger benign")
+
+        with pytest.raises(RuntimeError, match="king benign"):
+            run_duel(_clips(n=2), king_benign, chall_benign, mse, master_seed=1,
+                     delta_threshold=0.0025, alpha=0.05, n_bootstrap=50, concurrent=True)
+
+    def test_when_both_are_fatal_the_propagated_one_is_still_recognized_as_fatal(self):
+        from leoma.eval.errors import is_cuda_fatal
+
+        def king_fatal(clip, seed):
+            raise RuntimeError("bus error")
+
+        def chall_fatal(clip, seed):
+            raise RuntimeError("segmentation fault")
+
+        try:
+            run_duel(_clips(n=2), king_fatal, chall_fatal, mse, master_seed=1,
+                     delta_threshold=0.0025, alpha=0.05, n_bootstrap=50, concurrent=True)
+            assert False, "expected an exception"
+        except RuntimeError as e:
+            assert is_cuda_fatal(e)
+
+
+class TestGeneratePairDirectly:
+    """Unit tests on _generate_pair itself, isolated from the rest of run_duel."""
+
+    def test_sequential_mode_is_unaffected_by_the_fix(self):
+        from leoma.eval.video_runner import Clip, GenParams, _generate_pair
+
+        clip = Clip(clip_index=0, first_frame=np.zeros((8, 8, 3), dtype="uint8"), prompt="p",
+                   truth_frames=np.zeros((4, 8, 8, 3), dtype="uint8"), params=GenParams())
+        king = lambda c, s: np.full((4, 8, 8, 3), 1, dtype="uint8")
+        chall = lambda c, s: np.full((4, 8, 8, 3), 2, dtype="uint8")
+
+        k, c = _generate_pair(king, chall, clip, 42, executor=None)
+        assert (k == 1).all() and (c == 2).all()
+
+    def test_concurrent_mode_with_no_errors_returns_both_results(self):
+        from concurrent.futures import ThreadPoolExecutor
+
+        from leoma.eval.video_runner import Clip, GenParams, _generate_pair
+
+        clip = Clip(clip_index=0, first_frame=np.zeros((8, 8, 3), dtype="uint8"), prompt="p",
+                   truth_frames=np.zeros((4, 8, 8, 3), dtype="uint8"), params=GenParams())
+        king = lambda c, s: np.full((4, 8, 8, 3), 1, dtype="uint8")
+        chall = lambda c, s: np.full((4, 8, 8, 3), 2, dtype="uint8")
+
+        with ThreadPoolExecutor(max_workers=2) as ex:
+            k, c = _generate_pair(king, chall, clip, 42, executor=ex)
+        assert (k == 1).all() and (c == 2).all()
+
+
 class TestNoThreadLeak:
     def test_the_executor_is_torn_down_after_every_run(self):
         baseline = threading.active_count()
diff --git a/tests/unit/test_devices.py b/tests/unit/test_devices.py
index f074b21..84a05a2 100644
--- a/tests/unit/test_devices.py
+++ b/tests/unit/test_devices.py
@@ -18,6 +18,54 @@ def test_disabled_means_no_devices_assigned(self):
         assert d.challenger_device is None
         assert d.concurrent is False
 
+    def test_a_single_override_with_concurrency_disabled_is_still_ignored(self):
+        """Only one side pinned isn't enough information to safely pick the other,
+        same rule as the concurrent case."""
+        d = resolve_duel_devices(
+            concurrent_enabled=False, cuda_device_count=8, king_device_override="cuda:2",
+        )
+        assert d.king_device is None
+        assert d.challenger_device is None
+        assert d.concurrent is False
+
+
+class TestPinningWithoutConcurrency:
+    """Pinning answers 'which devices to load onto'; concurrency answers 'do the two
+    generations overlap in time'. An operator running several eval-server PROCESSES on
+    one multi-GPU box wants each process's king+challenger spread across its own
+    device pair even if that process still runs its two generations sequentially — so
+    an explicit pair of overrides must be honored regardless of the concurrency flag.
+    """
+
+    def test_both_overrides_are_honored_for_loading_even_with_concurrency_off(self):
+        d = resolve_duel_devices(
+            concurrent_enabled=False, cuda_device_count=8,
+            king_device_override="cuda:2", challenger_device_override="cuda:3",
+        )
+        assert d.king_device == "cuda:2"
+        assert d.challenger_device == "cuda:3"
+        assert d.concurrent is False   # generation is still sequential — not requested
+        assert "sequentially" in d.note
+
+    def test_identical_overrides_with_concurrency_off_still_load_there(self):
+        d = resolve_duel_devices(
+            concurrent_enabled=False, cuda_device_count=8,
+            king_device_override="cuda:0", challenger_device_override="cuda:0",
+        )
+        assert d.king_device == d.challenger_device == "cuda:0"
+        assert d.concurrent is False
+
+    def test_pinning_without_concurrency_ignores_the_reported_device_count(self):
+        """Loading is not a claim about overlap, so it doesn't need >=2 real devices —
+        it works even (degenerately) on a single-GPU box that just wants a named
+        device rather than whatever 'cuda' defaults to."""
+        d = resolve_duel_devices(
+            concurrent_enabled=False, cuda_device_count=1,
+            king_device_override="cuda:0", challenger_device_override="cuda:0",
+        )
+        assert d.king_device == "cuda:0"
+        assert d.concurrent is False
+
 
 class TestInsufficientDevices:
     @pytest.mark.parametrize("count", [0, 1])