Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,10 @@ This file defines how coding agents should work in this repository.
explicit custom IDs locally before service RPC, stop-all fallback, or daemon
side effects. Only omitting the option means all-session status or no stop
target was chosen.
- `keep-gpu stop --all` daemon fallback may signal an ownership-verified daemon
only after typed service transport failures such as `ServiceUnreachableError`;
generic application/runtime errors must never be classified by message text
such as `timed out`.
- For CLI `--gpu-ids`, only omission means all visible GPUs; explicit empty or
whitespace-only values are invalid and must not silently expand to all GPUs.
- Blocking CLI mode must defer omitted-GPU hardware enumeration to
Expand Down
4 changes: 4 additions & 0 deletions docs/reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,10 @@ boundary first, waits only for starting jobs in that boundary, and does not stop
later starts.
`--all` releases the sessions in its snapshot concurrently and prints results
in deterministic snapshot order with the same additive response fields.
If the stop RPC transport is unreachable, `--all` may force-stop an
ownership-verified local daemon. Application/runtime errors from the service are
reported as JSON errors and do not trigger daemon stop fallback based on message
text.
The output is a directly parseable JSON object, including `{"error": "..."}` for
service/runtime errors after CLI parsing succeeds. Malformed JSON-RPC service
envelopes, including missing or non-string `error.message`, and malformed stop
Expand Down
7 changes: 1 addition & 6 deletions src/keep_gpu/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -985,12 +985,7 @@ def _rollback_auto_started_service_on_startup_unavailable(


def _is_service_unreachable_error(exc: RuntimeError) -> bool:
if isinstance(exc, ServiceUnreachableError):
return True
if isinstance(exc, (ServiceRPCError, ServiceResponseError)):
return False
message = str(exc).lower()
return "cannot reach keepgpu service" in message or "timed out" in message
return isinstance(exc, ServiceUnreachableError)


def _stop_all_sessions_with_fallback(host: str, port: int) -> Dict[str, Any]:
Expand Down
47 changes: 41 additions & 6 deletions tests/test_cli_service_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -2761,7 +2761,7 @@ def test_service_stop_refuses_active_sessions_without_force(monkeypatch):

def test_stop_handles_service_timeout_without_traceback(monkeypatch):
def fake_rpc(method, params, host, port, timeout=8.0):
raise RuntimeError(
raise cli.ServiceUnreachableError(
"Cannot reach KeepGPU service at http://127.0.0.1:8765/rpc: timed out"
)

Expand All @@ -2784,7 +2784,9 @@ def test_cli_module_avoids_eager_gpu_imports():

def test_stop_all_fallback_force_stops_managed_daemon(monkeypatch):
def fake_rpc(method, params, host, port, timeout=8.0):
raise RuntimeError("timed out")
raise cli.ServiceUnreachableError(
"Cannot reach KeepGPU service at http://127.0.0.1:8765/rpc: timed out"
)

monkeypatch.setattr(cli, "_rpc_call", fake_rpc)
monkeypatch.setattr(cli, "_read_service_pid", lambda host, port: 1234)
Expand All @@ -2802,11 +2804,44 @@ def fake_rpc(method, params, host, port, timeout=8.0):
assert payload["errors"] == {}


def test_stop_all_does_not_fallback_for_rpc_application_error(monkeypatch):
def test_stop_all_does_not_fallback_for_generic_timeout_error(monkeypatch):
called = {"stop_process": False}

def fake_rpc(method, params, host, port, timeout=8.0):
raise RuntimeError("controller timed out while releasing session")

def fake_stop_process(host, port):
called["stop_process"] = True
return True

monkeypatch.setattr(cli, "_rpc_call", fake_rpc)
monkeypatch.setattr(cli, "_read_service_pid", lambda host, port: 1234)
monkeypatch.setattr(cli, "_pid_alive", lambda pid: True)
monkeypatch.setattr(cli, "_stop_service_process", fake_stop_process)

result = runner.invoke(cli.app, ["stop", "--all"])

assert result.exit_code == 1
assert "controller timed out while releasing session" in result.output
assert "force-stopped local daemon" not in result.output
assert called["stop_process"] is False


@pytest.mark.parametrize(
("exc", "message"),
[
(RuntimeError("validation failed"), "validation failed"),
(cli.ServiceRPCError("rpc failed"), "rpc failed"),
(cli.ServiceResponseError("malformed response"), "malformed response"),
],
)
def test_stop_all_does_not_fallback_for_rpc_application_error(
monkeypatch, exc, message
):
called = {"stop_process": False}

def fake_rpc(method, params, host, port, timeout=8.0):
raise RuntimeError("validation failed")
raise exc

def fake_stop_process(host, port):
called["stop_process"] = True
Expand All @@ -2820,14 +2855,14 @@ def fake_stop_process(host, port):
result = runner.invoke(cli.app, ["stop", "--all"])

assert result.exit_code == 1
assert "validation failed" in result.output
assert message in result.output
assert "force-stopped local daemon" not in result.output
assert called["stop_process"] is False


def test_stop_all_fallback_requires_stop_process_success(monkeypatch):
def fake_rpc(method, params, host, port, timeout=8.0):
raise RuntimeError(
raise cli.ServiceUnreachableError(
"Cannot reach KeepGPU service at http://127.0.0.1:8765/rpc: timed out"
)

Expand Down
Loading