[global] fix: clean failed startup workers

AGENTS.md

@@ -374,6 +374,9 @@ This file defines how coding agents should work in this repository.
 - Failed single-GPU startup paths must clear any vendor-library initialization
   and stale `_thread`/`_stop_evt` state before re-raising unless a real worker is
   still alive and explicitly stopping.
+- `GlobalGPUController.keep()` must best-effort release a child controller that
+  fails after leaving `_thread` or `_stop_evt` worker state, then roll back
+  previously-started children while preserving the original startup error.
 - Internal single-GPU startup paths that receive a `startup_evt` must always
   signal it before returning, and paths without a `startup_errors` list must
   retain the failure detail in `allocation_status()`.

docs/concepts/architecture.md

@@ -43,8 +43,9 @@ CLI args ──▶ GlobalGPUController ──▶ [backend controller rank=0]
 1. The CLI (or your Python code) instantiates `GlobalGPUController`; invalid
    local inputs fail before backend discovery.
 2. During `keep()` / `__enter__`, `GlobalGPUController` starts each worker.
-   If a later worker fails to start, already-started workers are released before
-   the original start error is re-raised.
+   If a later worker fails to start, any failed worker that left partial worker
+   state is best-effort released, then already-started workers are released
+   before the original start error is re-raised.
 3. Each CUDA worker:
    - Has already validated its direct visible `rank` against the current CUDA
      device count before `torch.device`, `set_device`, telemetry, or allocation

src/keep_gpu/global_gpu_controller/global_gpu_controller.py

@@ -20,6 +20,13 @@
 logger = setup_logger(__name__)
 
 
+def _controller_has_worker_state(ctrl) -> bool:
+    return (
+        getattr(ctrl, "_thread", None) is not None
+        or getattr(ctrl, "_stop_evt", None) is not None
+    )
+
+
 class ControllerStartupUnavailable(Exception):
     """Expected hardware/platform unavailability during controller startup."""
 
@@ -139,6 +146,16 @@ def keep(self) -> None:
             try:
                 ctrl.keep()
             except Exception:
+                if _controller_has_worker_state(ctrl):
+                    try:
+                        ctrl.release()
+                    except Exception as cleanup_exc:
+                        logger.warning(
+                            "Failed to clean up failed controller rank %s after start "
+                            "failure: %s",
+                            getattr(ctrl, "rank", "unknown"),
+                            cleanup_exc,
+                        )
                 for started_ctrl in reversed(started):
                     try:
                         started_ctrl.release()

tests/global_controller/global_keep_test.py

@@ -65,6 +65,94 @@ def release(self):
     assert instances[1].released is False
 
 
+def test_global_keep_releases_failed_controller_with_worker_state(monkeypatch):
+    monkeypatch.setattr(pm, "_cached_platform", pm.ComputingPlatform.CUDA)
+    monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+
+    instances = []
+
+    class DummyThread:
+        @staticmethod
+        def is_alive():
+            return True
+
+    class DummyController:
+        def __init__(self, *, rank, interval, vram_to_keep, busy_threshold):
+            self.rank = rank
+            self.kept = False
+            self.released = False
+            self._thread = None
+            self._stop_evt = None
+            instances.append(self)
+
+        def keep(self):
+            if self.rank == 1:
+                self._thread = DummyThread()
+                self._stop_evt = object()
+                raise RuntimeError("rank 1 failed after spawning worker")
+            self.kept = True
+
+        def release(self):
+            self.released = True
+
+    import keep_gpu.single_gpu_controller.cuda_gpu_controller as cuda_module
+
+    monkeypatch.setattr(cuda_module, "CudaGPUController", DummyController)
+
+    controller = GlobalGPUController(gpu_ids=[0, 1], vram_to_keep="8MB")
+
+    with pytest.raises(RuntimeError, match="rank 1 failed after spawning worker"):
+        controller.keep()
+
+    assert instances[0].released is True
+    assert instances[1].released is True
+
+
+def test_global_keep_preserves_start_error_when_failed_child_cleanup_fails(
+    monkeypatch,
+):
+    monkeypatch.setattr(pm, "_cached_platform", pm.ComputingPlatform.CUDA)
+    monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+
+    instances = []
+
+    class DummyThread:
+        @staticmethod
+        def is_alive():
+            return True
+
+    class DummyController:
+        def __init__(self, *, rank, interval, vram_to_keep, busy_threshold):
+            self.rank = rank
+            self.released = False
+            self._thread = None
+            self._stop_evt = None
+            instances.append(self)
+
+        def keep(self):
+            if self.rank == 1:
+                self._thread = DummyThread()
+                self._stop_evt = object()
+                raise RuntimeError("original start failure")
+
+        def release(self):
+            self.released = True
+            if self.rank == 1:
+                raise RuntimeError("cleanup failed")
+
+    import keep_gpu.single_gpu_controller.cuda_gpu_controller as cuda_module
+
+    monkeypatch.setattr(cuda_module, "CudaGPUController", DummyController)
+
+    controller = GlobalGPUController(gpu_ids=[0, 1], vram_to_keep="8MB")
+
+    with pytest.raises(RuntimeError, match="original start failure"):
+        controller.keep()
+
+    assert instances[0].released is True
+    assert instances[1].released is True
+
+
 def test_global_controller_rejects_zero_visible_cuda_devices(monkeypatch):
     monkeypatch.setattr(pm, "_cached_platform", pm.ComputingPlatform.CUDA)
     monkeypatch.setattr(torch.cuda, "device_count", lambda: 0)