From afd29bba71afe88fbf8ad6ce0eb68e7fb513fde1 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Wed, 4 Nov 2026 15:26:54 -0700 Subject: [PATCH 1/5] fix: preserve confirmed exit evidence across adapter health checks Signed-off-by: Sara Tadayon --- .../src/nemo_fabric_adapters/codex/adapter.py | 4 +- adapters/typescript/cline/src/cline-sdk.ts | 7 +- .../typescript/cline/test/health.test.mjs | 43 ++++ .../common/test/fixtures/acp-agent.mjs | 4 + adapters/typescript/droid/README.md | 5 +- adapters/typescript/droid/src/droid-sdk.ts | 30 ++- .../typescript/droid/test/droid-sdk.test.mjs | 45 ++++ .../typescript/droid/test/health.test.mjs | 20 ++ adapters/typescript/kilo/src/kilo-sdk.ts | 30 ++- .../typescript/kilo/test/health-sdk.test.mjs | 30 ++- adapters/typescript/kilo/test/health.test.mjs | 20 ++ adapters/typescript/kiro/test/health.test.mjs | 20 ++ .../typescript/kiro/test/kiro-acp.test.mjs | 14 ++ .../typescript/opencode/test/health.test.mjs | 20 ++ adapters/typescript/pi/test/health.test.mjs | 20 ++ adapters/typescript/qwen/src/qwen-sdk.ts | 10 +- adapters/typescript/qwen/test/health.test.mjs | 60 +++++- .../typescript/strands/test/health.test.mjs | 20 ++ .../strands/test/strands-acp.test.mjs | 21 ++ crates/fabric-core/src/check.rs | 39 +++- crates/fabric-core/src/resource_snapshot.rs | 47 ++++ crates/fabric-core/src/runtime.rs | 204 ++++++++++++++++++ docs/integrations/harness/droid.mdx | 5 +- docs/sdk/health-checks.mdx | 4 + tests/adapters/test_claude_health.py | 27 ++- tests/adapters/test_codex_adapter.py | 38 ++++ tests/adapters/test_deepagents_health.py | 8 +- tests/adapters/test_hermes_health.py | 8 +- tests/adapters/test_openclaw_health.py | 15 +- tests/adapters/test_openhands.py | 6 +- tests/e2e/test_hermes_checks.py | 67 ++++++ 31 files changed, 834 insertions(+), 57 deletions(-) diff --git a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py index 90bdaddaa..a3b371a90 100644 --- a/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py +++ b/adapters/python/codex/src/nemo_fabric_adapters/codex/adapter.py @@ -1483,13 +1483,13 @@ def observed(outcome: str, reason: str | None) -> dict[str, Any]: or native._sync is not transport or transport._proc is not process or thread.id != thread_id - or self._unusable != unusable - or client._initialized != initialized or thread._codex is not thread_owner ): return {**result, "reason_code": "observation_changed"} if process.poll() is not None: outcome, reason = "failed", "agent_exited" + elif self._unusable != unusable or client._initialized != initialized: + return {**result, "reason_code": "observation_changed"} return { **result, "outcome": outcome, diff --git a/adapters/typescript/cline/src/cline-sdk.ts b/adapters/typescript/cline/src/cline-sdk.ts index fbf468412..4fba901d1 100644 --- a/adapters/typescript/cline/src/cline-sdk.ts +++ b/adapters/typescript/cline/src/cline-sdk.ts @@ -577,7 +577,7 @@ class ClineSdkSessionHandle implements ClineSessionHandle { } const unchanged = () => !this.stopped && residentSessions(this.core) === this.sessions && this.sessions?.get(sessionId) === resident && resident.agent === this.agent && - resident.agent.getAgentId() === this.agentId && !resident.agent.shutdownCalled; + resident.agent.getAgentId() === this.agentId; try { const sessions = residentSessions(this.core); const current = sessions?.get(sessionId); @@ -608,14 +608,17 @@ class ClineSdkSessionHandle implements ClineSessionHandle { this.probing = false; } if (!unchanged()) return unobservedHealth("observation_changed"); + if (resident.agent.shutdownCalled) return observation("failed", "session_closed"); if (snapshot?.sessionId !== sessionId || !["idle", "running", "pending"].includes(snapshot.status)) { return observation("failed", "health_failed"); } } - return unchanged() ? observation("passed", null) : unobservedHealth("observation_changed"); + if (!unchanged()) return unobservedHealth("observation_changed"); + return resident.agent.shutdownCalled ? observation("failed", "session_closed") : observation("passed", null); } catch { try { if (!unchanged()) return unobservedHealth("observation_changed"); + if (resident.agent.shutdownCalled) return observation("failed", "session_closed"); } catch { return unobservedHealth("observation_changed"); } return unobservedHealth("probe_error"); } diff --git a/adapters/typescript/cline/test/health.test.mjs b/adapters/typescript/cline/test/health.test.mjs index 4a1a426cc..5b2dde9f1 100644 --- a/adapters/typescript/cline/test/health.test.mjs +++ b/adapters/typescript/cline/test/health.test.mjs @@ -205,3 +205,46 @@ test("a replaced native host cannot turn a missing resident into confirmed closu assert.equal((await handle.check("live")).reason_code, "observation_changed"); await handle.stop(); }); + +test("runtime health preserves confirmed native exit evidence without invoking or stopping", async (t) => { + const evidence = { + outcome: "failed", reason_code: "agent_exited", activity: "stopped", + source: "fabric.adapter.liveness", observed_at: new Date().toISOString(), + }; + let stops = 0; + const runtime = new ClineAdapterRuntime({ + create: async () => ({ + id: "native-session", + check: async () => evidence, + prompt: async () => { throw new Error("health must not invoke"); }, + stop: async () => { stops += 1; }, + }), + }); + await runtime.start(input); + t.after(() => runtime.stop()); + assert.deepEqual(await runtime.check("live"), evidence); + assert.equal(stops, 0); +}); + +for (const rejected of [false, true]) { + test(`ready preserves the same native agent's closure when get ${rejected ? "rejects" : "resolves"}`, async (t) => { + const fixture = nativeFixture(); + const reply = Promise.withResolvers(); + fixture.core.get = () => reply.promise; + const handle = await new ClineSdkSessionFactory(fixture.loader).create(input); + t.after(() => handle.stop()); + const checking = handle.check("ready"); + fixture.agent.shutdownCalled = true; + if (rejected) reply.reject(new Error("PRIVATE native shutdown")); + else reply.resolve({ sessionId: "session", status: "idle" }); + const result = await checking; + assert.equal(result.outcome, "failed"); + assert.equal(result.reason_code, "session_closed"); + assert.equal(result.activity, "stopped"); + assert.ok(Number.isFinite(Date.parse(result.observed_at))); + assert.doesNotMatch(JSON.stringify(result), /PRIVATE/); + assert.equal(fixture.calls.send, 0); + assert.equal(fixture.sessions.get("session"), fixture.resident); + assert.equal(fixture.resident.agent, fixture.agent); + }); +} diff --git a/adapters/typescript/common/test/fixtures/acp-agent.mjs b/adapters/typescript/common/test/fixtures/acp-agent.mjs index d74ea61a6..3833ca971 100644 --- a/adapters/typescript/common/test/fixtures/acp-agent.mjs +++ b/adapters/typescript/common/test/fixtures/acp-agent.mjs @@ -16,6 +16,10 @@ let effort = "medium"; let mcpCount = 0; let ready = false; +if (process.env.TEST_PROCESS_MARKER !== undefined) { + writeFileSync(process.env.TEST_PROCESS_MARKER, String(process.pid)); +} + if (process.env.TEST_DESCENDANT_MARKER !== undefined) { const descendant = spawn(process.execPath, ["-e", ` process.on("SIGTERM", () => {}); diff --git a/adapters/typescript/droid/README.md b/adapters/typescript/droid/README.md index 994c3c0d0..7f47f9c49 100644 --- a/adapters/typescript/droid/README.md +++ b/adapters/typescript/droid/README.md @@ -115,7 +115,10 @@ settings, and unknown built-in tool IDs fail instead of being ignored. `Fabric.check()` supports `live`. The adapter sends a fresh native `DroidSession.listTools()` request. A successful SDK response establishes that the selected Droid subprocess responds; a retained SDK handle is insufficient. -A failed request returns unknown health and does not close the session. +A failed request returns unknown health unless the SDK reports `ProcessExitError`, +which confirms that the owned subprocess exited and returns `failed/agent_exited`. +Later checks preserve that terminal observation and its original `observed_at`. +Checks do not close the session. `active` and `ready` are unsupported. The pinned SDK does not expose a qualified read-only probe for prompt acceptance or native readiness. A busy invocation diff --git a/adapters/typescript/droid/src/droid-sdk.ts b/adapters/typescript/droid/src/droid-sdk.ts index 9d411e402..d6464436f 100644 --- a/adapters/typescript/droid/src/droid-sdk.ts +++ b/adapters/typescript/droid/src/droid-sdk.ts @@ -133,6 +133,7 @@ interface DroidSdkSession { interface DroidSdkModule { createSession(options: Record): Promise; + ProcessExitError?: typeof import("@factory/droid-sdk/node").ProcessExitError; } interface EnvironmentLease { @@ -688,8 +689,13 @@ class DroidSdkSessionHandle implements DroidSessionHandle { private readonly runtimeRoot?: string; private stopped = false; private healthPending = false; + private processExit?: AdapterCheckResult; - constructor(session: DroidSdkSession, runtimeRoot?: string) { + constructor( + session: DroidSdkSession, + runtimeRoot?: string, + private readonly processExitError?: DroidSdkModule["ProcessExitError"], + ) { this.session = session; this.runtimeRoot = runtimeRoot; } @@ -706,6 +712,7 @@ class DroidSdkSessionHandle implements DroidSessionHandle { ...unobservedHealth("unsupported_level"), outcome: "unsupported", }; + if (this.processExit !== undefined) return { ...this.processExit }; // The SDK has no cancellation option for listTools. Retain at most one pending request across caller timeouts. if (this.healthPending) return unobservedHealth("probe_timeout"); this.healthPending = true; @@ -720,10 +727,19 @@ class DroidSdkSessionHandle implements DroidSessionHandle { source: "fabric.adapter.liveness", observed_at: new Date().toISOString(), }; - } catch { - return unobservedHealth( - this.stopped ? "observation_changed" : "probe_error", - ); + } catch (error) { + if (this.stopped) return unobservedHealth("observation_changed"); + if (this.processExitError !== undefined && error instanceof this.processExitError) { + this.processExit = { + outcome: "failed", + reason_code: "agent_exited", + activity: "stopped", + source: "fabric.adapter.liveness", + observed_at: new Date().toISOString(), + }; + return { ...this.processExit }; + } + return unobservedHealth("probe_error"); } finally { this.healthPending = false; } @@ -820,10 +836,12 @@ export class DroidSdkSessionFactory implements DroidSessionFactory { }; let session: DroidSdkSession | undefined; + let processExitError: DroidSdkModule["ProcessExitError"]; try { const environment = leaseEnvironment(sessionEnvironment, apiKeyEnv, apiKey, this.parentEnvironment); try { const sdk = await this.loader(); + processExitError = sdk.ProcessExitError; session = await sdk.createSession({ cwd: workspace, modelId: selected.model, @@ -846,7 +864,7 @@ export class DroidSdkSessionFactory implements DroidSessionFactory { await applyToolPolicy(session, input.config, mcpServers); await validateMcpTools(session, mcpServers, this.mcpStartupTimeoutMs); await validateSkills(session, runtimeProfile.entrypoints); - return new DroidSdkSessionHandle(session, runtimeProfile.root); + return new DroidSdkSessionHandle(session, runtimeProfile.root, processExitError); } catch (error) { await session?.close().catch(() => undefined); if (runtimeProfile.root !== undefined) { diff --git a/adapters/typescript/droid/test/droid-sdk.test.mjs b/adapters/typescript/droid/test/droid-sdk.test.mjs index 0a7d060c3..e673a70a4 100644 --- a/adapters/typescript/droid/test/droid-sdk.test.mjs +++ b/adapters/typescript/droid/test/droid-sdk.test.mjs @@ -7,6 +7,8 @@ import { tmpdir } from "node:os"; import { basename, dirname, join } from "node:path"; import test from "node:test"; +import { ProcessExitError } from "@factory/droid-sdk/node"; + import { DroidSdkSessionFactory } from "../dist/droid-sdk.js"; function input(overrides = {}) { @@ -547,3 +549,46 @@ test("native health failures remain unknown and never close a session", async () assert.equal((await handle.prompt("one")).status, "completed"); await handle.stop(); }); + +test("health retains the SDK's confirmed process exit without repeating requests", async (t) => { + const sdk = fakeSdk(); + const loader = async () => ({ ...await sdk.loader(), ProcessExitError }); + const handle = await new DroidSdkSessionFactory(loader).create(input()); + t.after(() => handle.stop()); + let calls = 0; + sdk.session.listTools = async () => { + calls += 1; + throw new ProcessExitError("PRIVATE-EXIT-DETAILS", { exitCode: 7 }); + }; + for (let attempt = 0; attempt < 2; attempt += 1) { + const result = await handle.check("live"); + assert.equal(result.outcome, "failed"); + assert.equal(result.reason_code, "agent_exited"); + assert.equal(result.activity, "stopped"); + assert.equal(result.source, "fabric.adapter.liveness"); + assert.ok(Number.isFinite(Date.parse(result.observed_at))); + assert.doesNotMatch(JSON.stringify(result), /PRIVATE/); + } + assert.equal(calls, 1); + assert.equal(sdk.calls.close, 0); + assert.deepEqual(sdk.calls.stream, []); +}); + +test("health leaves transport errors and same-named errors inconclusive", async (t) => { + const sdk = fakeSdk(); + const loader = async () => ({ ...await sdk.loader(), ProcessExitError }); + const handle = await new DroidSdkSessionFactory(loader).create(input()); + t.after(() => handle.stop()); + const misleading = new Error("PRIVATE process exited"); + misleading.name = "ProcessExitError"; + for (const error of [new Error("PRIVATE connection lost"), misleading]) { + sdk.session.listTools = async () => { throw error; }; + const result = await handle.check("live"); + assert.equal(result.outcome, "unknown"); + assert.equal(result.reason_code, "probe_error"); + assert.equal(result.observed_at, null); + assert.doesNotMatch(JSON.stringify(result), /PRIVATE/); + } + assert.equal(sdk.calls.close, 0); + assert.deepEqual(sdk.calls.stream, []); +}); diff --git a/adapters/typescript/droid/test/health.test.mjs b/adapters/typescript/droid/test/health.test.mjs index f1cd5338c..09c0cd144 100644 --- a/adapters/typescript/droid/test/health.test.mjs +++ b/adapters/typescript/droid/test/health.test.mjs @@ -184,3 +184,23 @@ test("a restart invalidates a check even when a factory reuses a session object" assert.equal((await checking).reason_code, "observation_changed"); await runtime.stop(); }); + +test("runtime health preserves confirmed native exit evidence without invoking or stopping", async (t) => { + const evidence = { + outcome: "failed", reason_code: "agent_exited", activity: "stopped", + source: "fabric.adapter.liveness", observed_at: new Date().toISOString(), + }; + let stops = 0; + const runtime = new DroidAdapterRuntime({ + create: async () => ({ + id: "native-session", + check: async () => evidence, + prompt: async () => { throw new Error("health must not invoke"); }, + stop: async () => { stops += 1; }, + }), + }); + await runtime.start(input); + t.after(() => runtime.stop()); + assert.deepEqual(await runtime.check("live"), evidence); + assert.equal(stops, 0); +}); diff --git a/adapters/typescript/kilo/src/kilo-sdk.ts b/adapters/typescript/kilo/src/kilo-sdk.ts index dc287bbcb..c3c5ae9f5 100644 --- a/adapters/typescript/kilo/src/kilo-sdk.ts +++ b/adapters/typescript/kilo/src/kilo-sdk.ts @@ -445,14 +445,14 @@ class KiloSdkSessionHandle implements KiloSessionHandle { outcome: "failed", activity: "stopped", }; - if (!this.server.isAlive()) - return { - outcome: "failed", - reason_code: "agent_exited", - activity: "stopped", - source: "fabric.adapter.liveness", - observed_at: new Date().toISOString(), - }; + const exited = (): AdapterCheckResult => ({ + outcome: "failed", + reason_code: "agent_exited", + activity: "stopped", + source: "fabric.adapter.liveness", + observed_at: new Date().toISOString(), + }); + if (!this.server.isAlive()) return exited(); if (level === "live") return { outcome: "passed", @@ -476,8 +476,8 @@ class KiloSdkSessionHandle implements KiloSessionHandle { options, ) : await this.client.global.health(options); - if (this.stopped || !this.server.isAlive()) - return unobservedHealth("observation_changed"); + if (this.stopped) return unobservedHealth("observation_changed"); + if (!this.server.isAlive()) return exited(); if (!isRecord(response.data)) return unobservedHealth("probe_error"); if (level === "active" && response.data.id !== this.id) return unobservedHealth("observation_changed"); @@ -506,13 +506,9 @@ class KiloSdkSessionHandle implements KiloSessionHandle { observed_at: new Date().toISOString(), }; } catch { - return unobservedHealth( - this.stopped || !this.server.isAlive() - ? "observation_changed" - : signal.aborted - ? "probe_timeout" - : "probe_error", - ); + if (this.stopped) return unobservedHealth("observation_changed"); + if (!this.server.isAlive()) return exited(); + return unobservedHealth(signal.aborted ? "probe_timeout" : "probe_error"); } finally { this.healthPending = false; } diff --git a/adapters/typescript/kilo/test/health-sdk.test.mjs b/adapters/typescript/kilo/test/health-sdk.test.mjs index c15dfef80..c63a3673b 100644 --- a/adapters/typescript/kilo/test/health-sdk.test.mjs +++ b/adapters/typescript/kilo/test/health-sdk.test.mjs @@ -172,7 +172,7 @@ test("native health request deadlines do not abort or close an invocation", asyn for (const level of ["active", "ready"]) { - test(`native server exit invalidates a rejected ${level} probe`, async (t) => { + test(`native server exit is preserved during a rejected ${level} probe`, async (t) => { const reply = Promise.withResolvers(); const { handle, state, client } = await session(t); if (level === "active") client.session.get = () => reply.promise; @@ -181,12 +181,34 @@ for (const level of ["active", "ready"]) { state.alive = false; reply.reject(new Error("PRIVATE-native-connection-error")); const result = await pending; - assert.equal(result.outcome, "unknown"); - assert.equal(result.reason_code, "observation_changed"); - assert.equal(result.observed_at, null); + assert.equal(result.outcome, "failed"); + assert.equal(result.reason_code, "agent_exited"); + assert.equal(result.activity, "stopped"); + assert.equal(result.source, "fabric.adapter.liveness"); + assert.ok(Number.isFinite(Date.parse(result.observed_at))); assert.equal(JSON.stringify(result).includes("PRIVATE"), false); assert.equal(state.aborts, 0); assert.equal(state.closed, 0); assert.equal((await handle.check("live")).reason_code, "agent_exited"); }); } + +for (const level of ["active", "ready"]) { + test(`preserves a confirmed Kilo process exit during a resolved ${level} request`, async (t) => { + const reply = Promise.withResolvers(); + const { handle, state, client } = await session(t); + client.session.get = () => reply.promise; + client.global.health = () => reply.promise; + const checking = handle.check(level); + state.alive = false; + reply.resolve({ data: { id: "session-1", healthy: true, version: "test" } }); + const result = await checking; + assert.equal(result.outcome, "failed"); + assert.equal(result.reason_code, "agent_exited"); + assert.equal(result.activity, "stopped"); + assert.equal(result.source, "fabric.adapter.liveness"); + assert.ok(Number.isFinite(Date.parse(result.observed_at))); + assert.equal(state.aborts, 0); + assert.equal(state.closed, 0); + }); +} diff --git a/adapters/typescript/kilo/test/health.test.mjs b/adapters/typescript/kilo/test/health.test.mjs index a855dc826..a533f5f38 100644 --- a/adapters/typescript/kilo/test/health.test.mjs +++ b/adapters/typescript/kilo/test/health.test.mjs @@ -188,3 +188,23 @@ test("a restart invalidates a check even when a factory reuses a session object" assert.equal((await checking).reason_code, "observation_changed"); await runtime.stop(); }); + +test("runtime health preserves confirmed native exit evidence without invoking or stopping", async (t) => { + const evidence = { + outcome: "failed", reason_code: "agent_exited", activity: "stopped", + source: "fabric.adapter.liveness", observed_at: new Date().toISOString(), + }; + let stops = 0; + const runtime = new KiloAdapterRuntime({ + create: async () => ({ + id: "native-session", + check: async () => evidence, + prompt: async () => { throw new Error("health must not invoke"); }, + stop: async () => { stops += 1; }, + }), + }); + await runtime.start(input); + t.after(() => runtime.stop()); + assert.deepEqual(await runtime.check("live"), evidence); + assert.equal(stops, 0); +}); diff --git a/adapters/typescript/kiro/test/health.test.mjs b/adapters/typescript/kiro/test/health.test.mjs index bb3fc5f63..b4c9d897f 100644 --- a/adapters/typescript/kiro/test/health.test.mjs +++ b/adapters/typescript/kiro/test/health.test.mjs @@ -184,3 +184,23 @@ test("a restart invalidates a check even when a factory reuses a session object" assert.equal((await checking).reason_code, "observation_changed"); await runtime.stop(); }); + +test("runtime health preserves confirmed native exit evidence without invoking or stopping", async (t) => { + const evidence = { + outcome: "failed", reason_code: "agent_exited", activity: "stopped", + source: "fabric.adapter.liveness", observed_at: new Date().toISOString(), + }; + let stops = 0; + const runtime = new KiroAdapterRuntime({ + create: async () => ({ + id: "native-session", + check: async () => evidence, + prompt: async () => { throw new Error("health must not invoke"); }, + stop: async () => { stops += 1; }, + }), + }); + await runtime.start(input); + t.after(() => runtime.stop()); + assert.deepEqual(await runtime.check("live"), evidence); + assert.equal(stops, 0); +}); diff --git a/adapters/typescript/kiro/test/kiro-acp.test.mjs b/adapters/typescript/kiro/test/kiro-acp.test.mjs index 7bb481a31..417b3a1e0 100644 --- a/adapters/typescript/kiro/test/kiro-acp.test.mjs +++ b/adapters/typescript/kiro/test/kiro-acp.test.mjs @@ -34,6 +34,7 @@ for (const fixtureType of process.platform === "win32" ? ["executable"] : ["shel TEST_ACP_FIXTURE: fixture, TEST_PROMPT_DELAY_MS: "100", TEST_PROMPT_STARTED: join(bin, "prompt-started"), + TEST_PROCESS_MARKER: join(bin, "native-pid"), }; const session = await new KiroAcpSessionFactory().create({ agentName: "kiro-test", @@ -108,6 +109,19 @@ for (const fixtureType of process.platform === "win32" ? ["executable"] : ["shel total_tokens: 5, }); } + const pid = Number(await readFile(environment.TEST_PROCESS_MARKER, "utf8")); + process.kill(pid, "SIGKILL"); + const exitDeadline = Date.now() + 2000; + let exited; + do { + exited = await session.check("live"); + if (exited.outcome === "failed") break; + assert.ok(Date.now() < exitDeadline, "native process exit was not observed"); + await new Promise((resolve) => setTimeout(resolve, 5)); + } while (true); + assert.equal(exited.reason_code, "agent_exited"); + assert.equal(exited.activity, "stopped"); + assert.ok(Number.isFinite(Date.parse(exited.observed_at))); } finally { await session.stop(); assert.equal((await session.check("live")).reason_code, "session_closed"); diff --git a/adapters/typescript/opencode/test/health.test.mjs b/adapters/typescript/opencode/test/health.test.mjs index 1ee3cf507..7a948ecbf 100644 --- a/adapters/typescript/opencode/test/health.test.mjs +++ b/adapters/typescript/opencode/test/health.test.mjs @@ -135,3 +135,23 @@ for (const fails of [false, true]) { } }); } + +test("runtime health preserves confirmed native exit evidence without invoking or stopping", async (t) => { + const evidence = { + outcome: "failed", reason_code: "agent_exited", activity: "stopped", + source: "fabric.adapter.liveness", observed_at: new Date().toISOString(), + }; + let stops = 0; + const runtime = new OpenCodeAdapterRuntime({ + create: async () => ({ + id: "native-session", + check: async () => evidence, + prompt: async () => { throw new Error("health must not invoke"); }, + stop: async () => { stops += 1; }, + }), + }); + await runtime.start(input); + t.after(() => runtime.stop()); + assert.deepEqual(await runtime.check("live"), evidence); + assert.equal(stops, 0); +}); diff --git a/adapters/typescript/pi/test/health.test.mjs b/adapters/typescript/pi/test/health.test.mjs index c29c9ecb3..4e354ccd9 100644 --- a/adapters/typescript/pi/test/health.test.mjs +++ b/adapters/typescript/pi/test/health.test.mjs @@ -297,3 +297,23 @@ test("native context liveness rejects disposed sessions while stored state remai assert.throws(() => context.isIdle()); assert.throws(() => session.extensionRunner.createContext().isIdle()); }); + +test("runtime health preserves confirmed native exit evidence without invoking or stopping", async (t) => { + const evidence = { + outcome: "failed", reason_code: "agent_exited", activity: "stopped", + source: "fabric.adapter.liveness", observed_at: new Date().toISOString(), + }; + let stops = 0; + const runtime = new PiAdapterRuntime({ + create: async () => ({ + id: "native-session", + check: async () => evidence, + prompt: async () => { throw new Error("health must not invoke"); }, + stop: async () => { stops += 1; }, + }), + }); + await runtime.start(input); + t.after(() => runtime.stop()); + assert.deepEqual(await runtime.check("live"), evidence); + assert.equal(stops, 0); +}); diff --git a/adapters/typescript/qwen/src/qwen-sdk.ts b/adapters/typescript/qwen/src/qwen-sdk.ts index d794e4f58..745e9c564 100644 --- a/adapters/typescript/qwen/src/qwen-sdk.ts +++ b/adapters/typescript/qwen/src/qwen-sdk.ts @@ -150,7 +150,8 @@ export class QwenSdkSession implements QwenSession { async check(level: CheckLevel): Promise { const source = level === "live" ? "fabric.adapter.liveness" : level === "active" ? "fabric.adapter.interface" : "fabric.adapter.readiness"; - if (this.closed || this.query.isClosed()) return { outcome: "failed", reason_code: "session_closed", source, observed_at: new Date().toISOString(), activity: "stopped" }; + const closure = (): AdapterCheckResult => ({ outcome: "failed", reason_code: "session_closed", source, observed_at: new Date().toISOString(), activity: "stopped" }); + if (this.closed || this.query.isClosed()) return closure(); if (this.healthObservation !== undefined) return { outcome: "unknown", reason_code: "probe_timeout", source, observed_at: null, activity: "unknown" }; const sessionId = this.query.getSessionId(); try { @@ -158,12 +159,15 @@ export class QwenSdkSession implements QwenSession { this.healthObservation = observation; void observation.finally(() => { if (this.healthObservation === observation) this.healthObservation = undefined; }).catch(() => { }); const valid = await observation; - if (this.closed || this.query.isClosed() || this.query.getSessionId() !== sessionId) return { outcome: "unknown", reason_code: "observation_changed", source, observed_at: null, activity: "unknown" }; + if (this.closed || this.query.getSessionId() !== sessionId) return { outcome: "unknown", reason_code: "observation_changed", source, observed_at: null, activity: "unknown" }; + if (this.query.isClosed()) return closure(); return valid ? { outcome: "passed", reason_code: null, source, observed_at: new Date().toISOString(), activity: "idle" } : { outcome: "unknown", reason_code: "probe_error", source, observed_at: new Date().toISOString(), activity: "unknown" }; } catch { - return { outcome: "unknown", reason_code: this.closed || this.query.isClosed() || this.query.getSessionId() !== sessionId ? "observation_changed" : "probe_error", source, observed_at: null, activity: "unknown" }; + if (this.closed || this.query.getSessionId() !== sessionId) return { outcome: "unknown", reason_code: "observation_changed", source, observed_at: null, activity: "unknown" }; + if (this.query.isClosed()) return closure(); + return { outcome: "unknown", reason_code: "probe_error", source, observed_at: null, activity: "unknown" }; } } diff --git a/adapters/typescript/qwen/test/health.test.mjs b/adapters/typescript/qwen/test/health.test.mjs index cdd492fdd..26a8ae7cb 100644 --- a/adapters/typescript/qwen/test/health.test.mjs +++ b/adapters/typescript/qwen/test/health.test.mjs @@ -112,7 +112,7 @@ test("later Qwen checks reject a delayed observation from an expired caller", as assert.equal(calls, 2); }); -test("Qwen validates native closure when the health request rejects", async () => { +test("Qwen preserves native closure when the health request rejects", async () => { const delayed = Promise.withResolvers(); let closed = false; const query = { isClosed: () => closed, getSessionId: () => "native", supportedCommands: () => delayed.promise }; @@ -121,8 +121,62 @@ test("Qwen validates native closure when the health request rejects", async () = closed = true; delayed.reject(new Error("PRIVATE native failure")); const result = await pending; - assert.equal(result.reason_code, "observation_changed"); - assert.equal(result.observed_at, null); + assert.equal(result.outcome, "failed"); + assert.equal(result.reason_code, "session_closed"); + assert.equal(result.activity, "stopped"); + assert.ok(Number.isFinite(Date.parse(result.observed_at))); assert.doesNotMatch(JSON.stringify(result), /PRIVATE/); assert.equal((await session.check("ready")).reason_code, "session_closed"); }); + +test("Qwen preserves native closure when a pending health request resolves", async () => { + const reply = Promise.withResolvers(); + let closed = false; + const query = { isClosed: () => closed, getSessionId: () => "native", supportedCommands: () => reply.promise }; + const session = new QwenSdkSession(query, {}, "unused", {}); + const checking = session.check("live"); + closed = true; + reply.resolve({ commands: [] }); + const result = await checking; + assert.equal(result.outcome, "failed"); + assert.equal(result.reason_code, "session_closed"); + assert.equal(result.activity, "stopped"); + assert.ok(Number.isFinite(Date.parse(result.observed_at))); +}); + +test("Qwen native replacement invalidates a previous session's closure", async () => { + const reply = Promise.withResolvers(); + let closed = false; + let id = "native"; + const query = { isClosed: () => closed, getSessionId: () => id, supportedCommands: () => reply.promise }; + const session = new QwenSdkSession(query, {}, "unused", {}); + const checking = session.check("live"); + closed = true; + id = "replacement"; + reply.reject(new Error("PRIVATE-OLD-SESSION")); + const result = await checking; + assert.equal(result.outcome, "unknown"); + assert.equal(result.reason_code, "observation_changed"); + assert.equal(result.observed_at, null); +}); + +test("runtime health preserves confirmed native exit evidence without invoking or stopping", async (t) => { + const evidence = { + outcome: "failed", reason_code: "agent_exited", activity: "stopped", + source: "fabric.adapter.liveness", observed_at: new Date().toISOString(), + }; + let stops = 0; + const runtime = new QwenAdapterRuntime({ + create: async () => ({ + id: "native-session", + check: async () => evidence, + prompt: async () => { throw new Error("health must not invoke"); }, + stop: async () => { stops += 1; }, + close: async () => { stops += 1; }, + }), + }); + await runtime.start(input); + t.after(() => runtime.stop()); + assert.deepEqual(await runtime.check("live"), evidence); + assert.equal(stops, 0); +}); diff --git a/adapters/typescript/strands/test/health.test.mjs b/adapters/typescript/strands/test/health.test.mjs index 1e50f5c5c..e3f9b1a8f 100644 --- a/adapters/typescript/strands/test/health.test.mjs +++ b/adapters/typescript/strands/test/health.test.mjs @@ -57,3 +57,23 @@ test("health sanitizes native errors without closing the session", async () => { assert.equal((await runtime.invoke({ input: "still usable" }, context)).status, "succeeded"); await runtime.stop(); }); + +test("runtime health preserves confirmed native exit evidence without invoking or stopping", async (t) => { + const evidence = { + outcome: "failed", reason_code: "agent_exited", activity: "stopped", + source: "fabric.adapter.liveness", observed_at: new Date().toISOString(), + }; + let stops = 0; + const runtime = new StrandsAdapterRuntime({ + create: async () => ({ + id: "native-session", + check: async () => evidence, + prompt: async () => { throw new Error("health must not invoke"); }, + stop: async () => { stops += 1; }, + }), + }); + await runtime.start(input); + t.after(() => runtime.stop()); + assert.deepEqual(await runtime.check("live"), evidence); + assert.equal(stops, 0); +}); diff --git a/adapters/typescript/strands/test/strands-acp.test.mjs b/adapters/typescript/strands/test/strands-acp.test.mjs index 36d6d9f80..ec3f6d8be 100644 --- a/adapters/typescript/strands/test/strands-acp.test.mjs +++ b/adapters/typescript/strands/test/strands-acp.test.mjs @@ -290,3 +290,24 @@ test("health uses the owned ACP child while the prompt remains blocked", { timeo assert.equal(stopped.outcome, "unknown"); assert.equal(stopped.reason_code, "runtime_unavailable"); }); + +test("health reports an owned ACP child exit before another invocation", { timeout: 10_000 }, async (t) => { + const input = await fixture(t); + const runtime = new StrandsAdapterRuntime(new StrandsAcpSessionFactory()); + await runtime.start(input); + t.after(() => runtime.stop()); + const { pid } = JSON.parse(await readFile(input.runtimeContext.environment.env.TEST_STRANDS_STATE, "utf8")); + assert.equal((await runtime.check("live")).outcome, "passed"); + process.kill(pid, "SIGKILL"); + const deadline = Date.now() + 2000; + let result; + do { + result = await runtime.check("live"); + if (result.outcome === "failed") break; + assert.ok(Date.now() < deadline, "native process exit was not observed"); + await delay(5); + } while (true); + assert.equal(result.reason_code, "agent_exited"); + assert.equal(result.activity, "stopped"); + assert.ok(Number.isFinite(Date.parse(result.observed_at))); +}); diff --git a/crates/fabric-core/src/check.rs b/crates/fabric-core/src/check.rs index 661b11685..091f9e1a7 100644 --- a/crates/fabric-core/src/check.rs +++ b/crates/fabric-core/src/check.rs @@ -956,11 +956,14 @@ pub fn check_environment( report.outcome = result.outcome; report.reason_code = result.reason_code; report.runtime = Some(RuntimeObservation { - state: match result.activity { - RuntimeActivity::Idle | RuntimeActivity::Busy => { + state: match (result.outcome, result.reason_code, result.activity) { + (CheckOutcome::Failed, Some(CheckReason::AgentExited), _) => { + CheckRuntimeState::Failed + } + (_, _, RuntimeActivity::Idle | RuntimeActivity::Busy) => { CheckRuntimeState::Active } - RuntimeActivity::Stopped => CheckRuntimeState::Stopped, + (_, _, RuntimeActivity::Stopped) => CheckRuntimeState::Stopped, _ => CheckRuntimeState::Unknown, }, activity: result.activity, @@ -1066,7 +1069,35 @@ fn verify_health_mac( } fn probe_runtime(runtime: &RuntimeHandle, level: CheckLevel, timeout: f64) -> AdapterCheckResult { - probe_target(&runtime.runtime_id, level, timeout, CheckScope::Runtime) + let started = Instant::now(); + // An exited owned host cannot serve its managed runtime, even if a native + // child or attached service survives. This does not establish descendant cleanup. + let has_exited = || crate::runtime::local_host_has_exited(&runtime.runtime_id) == Some(true); + let exit_observation = || AdapterCheckResult { + outcome: CheckOutcome::Failed, + reason_code: Some(CheckReason::AgentExited), + activity: RuntimeActivity::Stopped, + source: CheckSource::AdapterLiveness, + observed_at: Some(utc_now()), + }; + let exited = has_exited(); + let remaining = timeout - started.elapsed().as_secs_f64(); + if remaining <= 0.0 { + return unobserved(CheckReason::ProbeTimeout); + } + if exited { + return exit_observation(); + } + let result = probe_target(&runtime.runtime_id, level, remaining, CheckScope::Runtime); + // A host can exit after the first read and take the health connection with it. + // Preserve fresh exit evidence without extending the probe's deadline. + if started.elapsed().as_secs_f64() < timeout + && has_exited() + && started.elapsed().as_secs_f64() < timeout + { + return exit_observation(); + } + result } fn probe_target( diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 914db81ba..cb4a12076 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -83,6 +83,12 @@ impl LocalHostProcess { self.identity.is_some() && platform::start_identity(self.pid) == self.identity } + /// Confirm that the recorded process exited, without locking its lifecycle host. + /// Unreadable process evidence must not be interpreted as an exit. + pub(crate) fn has_exited(&self) -> Option { + platform::has_exited(self.pid, self.identity?) + } + /// Record the host `pid`, started by NeMo Fabric at `started_at`. pub(crate) fn new(pid: u32, started_at: SystemTime) -> Self { Self { @@ -525,6 +531,14 @@ mod platform { stat(pid).ok().map(|stat| stat.start) } + pub(super) fn has_exited(pid: u32, identity: u64) -> Option { + match stat(pid) { + Ok(stat) => Some(stat.start != identity || stat.zombie), + Err(ReadError::Gone) => Some(true), + Err(ReadError::Unavailable(_)) => None, + } + } + fn sysconf(name: libc::c_int) -> u64 { // SAFETY: sysconf has no memory-safety preconditions. u64::try_from(unsafe { libc::sysconf(name) }).unwrap_or(0) @@ -707,6 +721,16 @@ mod platform { usage(pid).ok().map(|info| info.ri_proc_start_abstime) } + pub(super) fn has_exited(pid: u32, identity: u64) -> Option { + match usage(pid) { + Ok(info) => { + Some(info.ri_proc_start_abstime != identity || info.ri_proc_exit_abstime != 0) + } + Err(ReadError::Gone) => Some(true), + Err(ReadError::Unavailable(_)) => None, + } + } + pub(super) fn read_tree(root: u32, identity: u64) -> Result { let timebase = timebase().ok_or(PROCESS_UNREADABLE)?; read_tree_with(&Live { timebase }, root, identity) @@ -723,6 +747,10 @@ mod platform { None } + pub(super) fn has_exited(_pid: u32, _identity: u64) -> Option { + None + } + pub(super) fn read_tree(_root: u32, _identity: u64) -> Result { Err("no process reader for this operating system") } @@ -786,6 +814,25 @@ mod tests { )); } + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn local_host_exit_requires_positive_process_identity_evidence() { + let mut host = LocalHostProcess::new(std::process::id(), SystemTime::now()); + assert_eq!(host.has_exited(), Some(false)); + let identity = host.identity.take().unwrap(); + assert_eq!( + host.has_exited(), + None, + "unreadable identity is not exit evidence" + ); + host.identity = Some(identity.wrapping_add(1)); + assert_eq!( + host.has_exited(), + Some(true), + "a reused PID belongs to another process" + ); + } + /// A host that exited but is not yet reaped is not a live process. #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d7041c9ed..271128120 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -2629,6 +2629,15 @@ fn local_host_processes() -> std::sync::MutexGuard<'static, BTreeMap Option { + let process = LOCAL_HOST_PROCESSES + .try_lock() + .ok()? + .get(runtime_id) + .cloned()?; + process.has_exited() +} + fn local_services() -> std::sync::MutexGuard<'static, BTreeMap>>> { LOCAL_SERVICES @@ -6438,6 +6447,201 @@ for line in sys.stdin: assert!(error.to_string().contains("request.overrides"), "{error}"); } + #[test] + #[cfg(any(target_os = "linux", target_os = "macos"))] + fn health_reports_exited_hosts_for_all_health_adapters_without_the_invocation_lock() { + use crate::check::{ + CheckLevel, CheckOutcome, CheckReason, CheckRequest, CheckRuntimeState, + }; + + let (root, plan) = local_host_plan("success"); + let mut runtime = start_runtime(&plan).expect("start local host"); + let original_adapter_id = runtime.adapter_id.clone(); + let mut adapters = Vec::new(); + let adapter_root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../adapters"); + for language in ["python", "typescript"] { + for entry in fs::read_dir(adapter_root.join(language)).unwrap() { + let directory = entry.unwrap().path(); + if !directory.is_dir() || directory.file_name().unwrap() == "node_modules" { + continue; + } + for file in fs::read_dir(directory).unwrap() { + let file = file.unwrap().path(); + if !file.to_string_lossy().ends_with(".fabric-adapter.json") { + continue; + } + let descriptor: Value = + serde_json::from_slice(&fs::read(file).unwrap()).unwrap(); + if let Some(levels) = descriptor["capabilities"]["metadata"]["health_checks"] + .as_array() + .filter(|levels| !levels.is_empty()) + { + adapters.push(( + descriptor["adapter_id"].as_str().unwrap().to_owned(), + serde_json::from_value::>(Value::Array(levels.clone())) + .unwrap(), + )); + } + } + } + } + assert!( + !adapters.is_empty(), + "health-capable adapter descriptors exist" + ); + // Third-party adapters must receive the same evidence without an ID allowlist. + adapters.push(("acme.fabric.health-test".into(), vec![CheckLevel::Live])); + let request = serde_json::from_value::(serde_json::json!({ + "scope": "runtime", "runtime_id": runtime.runtime_id, + "level": "ready", "timeout_seconds": 0.5, + })) + .unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + // An invocation owns this lock. Health must not acquire the lock. + let mut host = host.lock().unwrap(); + for (adapter_id, levels) in &adapters { + runtime.adapter_id = Some(adapter_id.clone()); + crate::check::register(&runtime, levels.clone()); + let before = crate::check::check_environment(&runtime.environment, &request).unwrap(); + assert_eq!(before.outcome, CheckOutcome::Unknown, "{adapter_id}"); + assert_eq!( + before.reason_code, + Some(CheckReason::ProbeError), + "{adapter_id}" + ); + } + + let process = local_host_processes() + .get(&runtime.runtime_id) + .unwrap() + .clone(); + host.child.kill().expect("kill the owned adapter host"); + let deadline = Instant::now() + Duration::from_secs(2); + while snapshot_local_host(&runtime.runtime_id, Some(&process)).is_ok() { + assert!(Instant::now() < deadline, "killed host did not exit"); + thread::sleep(Duration::from_millis(5)); + } + // Cover both an unreaped exited process and one whose PID is gone. + for reaped in [false, true] { + if reaped { + host.child.wait().expect("reap exited host"); + } + for (adapter_id, levels) in &adapters { + runtime.adapter_id = Some(adapter_id.clone()); + crate::check::register(&runtime, levels.clone()); + let report = + crate::check::check_environment(&runtime.environment, &request).unwrap(); + assert_eq!( + report.outcome, + CheckOutcome::Failed, + "{adapter_id}: {report:?}" + ); + assert_eq!( + report.reason_code, + Some(CheckReason::AgentExited), + "{adapter_id}" + ); + assert_eq!( + report.runtime.unwrap().state, + CheckRuntimeState::Failed, + "{adapter_id}" + ); + assert_eq!( + report.checks.len(), + 1, + "later levels must not run: {adapter_id}" + ); + assert_eq!(report.checks[0].name, crate::check::CheckStage::Live); + assert_eq!( + report.checks[0].source, + crate::check::CheckSource::AdapterLiveness + ); + assert!(report.checks[0].observed_at.is_some()); + assert!(report.elapsed_ms < 500.0); + } + } + // Without retained process identity, a failed connection remains inconclusive. + local_host_processes().remove(&runtime.runtime_id); + crate::check::register(&runtime, vec![CheckLevel::Live]); + let missing = crate::check::check_environment(&runtime.environment, &request).unwrap(); + assert_eq!(missing.outcome, CheckOutcome::Unknown); + assert_eq!(missing.reason_code, Some(CheckReason::ProbeError)); + local_host_processes().insert(runtime.runtime_id.clone(), process); + crate::check::register(&runtime, vec![]); + let unsupported = crate::check::check_environment(&runtime.environment, &request).unwrap(); + assert_eq!(unsupported.outcome, CheckOutcome::Unsupported); + assert_eq!( + unsupported.reason_code, + Some(CheckReason::UnsupportedAdapter) + ); + assert!(unsupported.checks.is_empty()); + crate::check::invalidate_runtime(&runtime); + runtime.adapter_id = original_adapter_id; + drop(host); + stop_runtime(&plan, &runtime).expect("clean up exited host"); + let _ = fs::remove_dir_all(root); + } + + #[test] + #[cfg(any(target_os = "linux", target_os = "macos"))] + fn health_preserves_a_host_exit_during_a_failed_probe() { + use crate::check::{CheckOutcome, CheckReason, CheckRequest}; + use std::os::unix::fs::PermissionsExt; + + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + crate::check::register(&runtime, vec![crate::check::CheckLevel::Live]); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let host = host.lock().unwrap(); + let pid = host.child.id(); + let process = local_host_processes() + .get(&runtime.runtime_id) + .unwrap() + .clone(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = local_host_directory(&runtime.runtime_id).join("health.endpoint.json"); + fs::write( + &endpoint, + serde_json::to_vec(&serde_json::json!({ + "protocol_version": "fabric.check/v1alpha1", + "runtime_id": runtime.runtime_id, + "port": listener.local_addr().unwrap().port(), + "token": "0123456789abcdef".repeat(4), + })) + .unwrap(), + ) + .unwrap(); + fs::set_permissions(&endpoint, fs::Permissions::from_mode(0o600)).unwrap(); + let exited = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut request = [0; 1024]; + assert!(stream.read(&mut request).unwrap() > 0); + // SAFETY: pid belongs to the still-owned host whose invocation lock is held. + assert_eq!(unsafe { libc::kill(pid as libc::pid_t, libc::SIGKILL) }, 0); + let deadline = Instant::now() + Duration::from_secs(2); + while process.has_exited() != Some(true) { + assert!(Instant::now() < deadline, "owned host did not exit"); + thread::sleep(Duration::from_millis(1)); + } + // Dropping the connection simulates a health channel lost with its host. + }); + let request = serde_json::from_value::(serde_json::json!({ + "scope": "runtime", "runtime_id": runtime.runtime_id, "timeout_seconds": 3.0, + })) + .unwrap(); + let report = crate::check::check_environment(&runtime.environment, &request).unwrap(); + exited.join().unwrap(); + drop(host); + stop_runtime(&plan, &runtime).unwrap(); + fs::remove_dir_all(root).unwrap(); + assert_eq!(report.outcome, CheckOutcome::Failed, "{report:?}"); + assert_eq!(report.reason_code, Some(CheckReason::AgentExited)); + assert!(report.checks[0].observed_at.is_some()); + } + #[test] fn local_host_crash_is_terminal_for_runtime() { let (root, plan) = local_host_plan("crash_after_start"); diff --git a/docs/integrations/harness/droid.mdx b/docs/integrations/harness/droid.mdx index 30b99a61b..c7df39830 100644 --- a/docs/integrations/harness/droid.mdx +++ b/docs/integrations/harness/droid.mdx @@ -214,7 +214,10 @@ streaming, Relay-backed streaming, custom model endpoints, or adapter-specific `Fabric.check()` supports `live`. The adapter sends a fresh native `DroidSession.listTools()` request. A successful SDK response establishes that the selected Droid subprocess responds; a retained SDK handle is insufficient. -A failed request returns unknown health and does not close the session. +A failed request returns unknown health unless the SDK reports `ProcessExitError`, +which confirms that the owned subprocess exited and returns `failed/agent_exited`. +Later checks preserve that terminal observation and its original `observed_at`. +Checks do not close the session. `active` and `ready` are unsupported. A busy invocation can remain live. Checks do not send prompts, validate model credentials, or diff --git a/docs/sdk/health-checks.mdx b/docs/sdk/health-checks.mdx index 695226c4b..0c1f60c6e 100644 --- a/docs/sdk/health-checks.mdx +++ b/docs/sdk/health-checks.mdx @@ -81,6 +81,10 @@ mini-SWE-agent, NOOA InteractiveAgent, NOOA BenchAgent, and Remote Agent adverti Pi readiness checks configured model, tool, and MCP state. Those observations do not perform a model request or execute a tool. OpenClaw readiness follows the Gateway's channel policies and can use the Gateway's cached observations. The linked OpenClaw endpoint tests use controlled Gateway HTTP responses. +On Linux and macOS, confirmed exit of a Fabric-owned adapter host returns `failed/agent_exited` for any adapter that advertises health checks. The managed runtime can no longer serve requests, even if a separate native child process or attached service survives. This observation does not establish descendant cleanup. An unreadable process identity or a failed health connection alone remains `unknown`. + +Adapter checks also preserve confirmed native-process exit and closed-session evidence when a native health request fails. A changed runtime or session identity still invalidates the observation and returns `unknown/observation_changed`. + Passing a health check does not establish successful inference, valid model credentials, provider availability, or successful tool execution unless the particular probe verifies that property. For example, Claude's readiness response does not require connected MCP servers, and Qwen's command response does not test model access. The linked qualification tests hold a local model response to verify that health checks can complete during invocation; those tests provide separate invocation evidence rather than changing what the health result means. ## Deadlines and Concurrent Invocations diff --git a/tests/adapters/test_claude_health.py b/tests/adapters/test_claude_health.py index 94d954bf2..5e58dc7d0 100644 --- a/tests/adapters/test_claude_health.py +++ b/tests/adapters/test_claude_health.py @@ -187,15 +187,28 @@ async def respond(): assert (result["outcome"], result["reason_code"]) == expected -async def test_claude_health_reports_probe_errors_without_claiming_process_death( - claude_health, +@pytest.mark.parametrize("level", ["active", "ready"]) +@pytest.mark.parametrize("exited", [False, True]) +async def test_claude_health_preserves_process_evidence_during_probe_errors( + claude_health, level, exited ): - runtime, client, _, _ = claude_health - client.get_mcp_status.side_effect = RuntimeError("secret-native-detail") - result = await runtime.check("active") - assert (result["outcome"], result["reason_code"]) == ("unknown", "probe_error") + runtime, client, _, process = claude_health + + async def failed_request(): + if exited: + process.returncode = 9 + raise RuntimeError("secret-native-detail") + + client.get_mcp_status.side_effect = failed_request + result = await runtime.check(level) + expected = ("failed", "agent_exited") if exited else ("unknown", "probe_error") + assert (result["outcome"], result["reason_code"]) == expected + assert result["activity"] == ("stopped" if exited else "idle") + assert result["observed_at"] is not None assert "secret-native-detail" not in str(result) - assert (await runtime.check("live"))["outcome"] == "passed" + assert (await runtime.check("live"))["outcome"] == ( + "failed" if exited else "passed" + ) async def test_claude_health_cancellation_bounds_native_requests_and_recovery( diff --git a/tests/adapters/test_codex_adapter.py b/tests/adapters/test_codex_adapter.py index 1128414ce..2a82da89f 100644 --- a/tests/adapters/test_codex_adapter.py +++ b/tests/adapters/test_codex_adapter.py @@ -2226,6 +2226,44 @@ async def test_health_detects_owned_process_exit(codex_health, level): thread.read.assert_not_awaited() +@pytest.mark.parametrize("level", ["active", "ready"]) +@pytest.mark.parametrize( + "change", ["unchanged", "unusable", "uninitialized", "process"] +) +@pytest.mark.parametrize("exited", [False, True]) +async def test_health_preserves_owned_process_exit_during_native_error( + codex_health, level, change, exited +): + runtime, client, thread, process = codex_health + + async def read(**kwargs): + if exited: + process.poll.return_value = 7 + if change == "unusable": + runtime._unusable = True + elif change == "uninitialized": + client._initialized = False + elif change == "process": + client._client._sync._proc = MagicMock(spec=subprocess.Popen) + raise RuntimeError("private-native-probe-detail") + + thread.read.side_effect = read + result = await runtime.check(level) + + if change == "process" or (not exited and change != "unchanged"): + expected = ("unknown", "observation_changed", "unknown") + elif exited: + expected = ("failed", "agent_exited", "stopped") + else: + expected = ("unknown", "probe_error", "idle") + assert (result["outcome"], result["reason_code"], result["activity"]) == expected + assert (result["observed_at"] is None) == ( + result["reason_code"] == "observation_changed" + ) + assert "private-native-probe-detail" not in json.dumps(result) + client.close.assert_not_awaited() + + @pytest.mark.parametrize( ("status", "level", "outcome", "reason"), [ diff --git a/tests/adapters/test_deepagents_health.py b/tests/adapters/test_deepagents_health.py index 6b1e07e29..9fc9d90d4 100644 --- a/tests/adapters/test_deepagents_health.py +++ b/tests/adapters/test_deepagents_health.py @@ -195,7 +195,10 @@ async def test_deepagents_unsupported_level_has_no_probe(deepagents_health): @pytest.mark.parametrize("change", ["close", "stop"]) -async def test_deepagents_health_detects_closure_during_read(deepagents_health, change): +@pytest.mark.parametrize("raises", [False, True]) +async def test_deepagents_health_detects_closure_during_read( + deepagents_health, change, raises +): runtime, agent, _, connection = deepagents_health async def changed(_config): @@ -203,6 +206,8 @@ async def changed(_config): connection._running = False else: runtime._stopping = True + if raises: + raise RuntimeError("private-native-probe-detail") return agent.aget_state.return_value agent.aget_state.side_effect = changed @@ -213,6 +218,7 @@ async def changed(_config): else ("unknown", "observation_changed") ) assert (result["outcome"], result["reason_code"]) == expected + assert "private-native-probe-detail" not in str(result) async def test_deepagents_health_distinguishes_cleanup_from_completed_stop( diff --git a/tests/adapters/test_hermes_health.py b/tests/adapters/test_hermes_health.py index 89e7971b2..35d366d7f 100644 --- a/tests/adapters/test_hermes_health.py +++ b/tests/adapters/test_hermes_health.py @@ -234,7 +234,10 @@ async def test_hermes_unsupported_level_has_no_probe(hermes_health): @pytest.mark.parametrize("change", ["close", "stop"]) -async def test_hermes_health_detects_closure_during_diagnostic(hermes_health, change): +@pytest.mark.parametrize("raises", [False, True]) +async def test_hermes_health_detects_closure_during_diagnostic( + hermes_health, change, raises +): runtime, agent = hermes_health def changed(): @@ -242,6 +245,8 @@ def changed(): agent._memory_provider_shutdown = True else: runtime._stopping = True + if raises: + raise RuntimeError("private-native-probe-detail") return agent.get_activity_summary.return_value agent.get_activity_summary.side_effect = changed @@ -252,6 +257,7 @@ def changed(): else ("unknown", "observation_changed") ) assert (result["outcome"], result["reason_code"]) == expected + assert "private-native-probe-detail" not in str(result) async def test_health_timeout_consumes_late_native_errors_without_logging( diff --git a/tests/adapters/test_openclaw_health.py b/tests/adapters/test_openclaw_health.py index fa805d503..35811569b 100644 --- a/tests/adapters/test_openclaw_health.py +++ b/tests/adapters/test_openclaw_health.py @@ -203,15 +203,17 @@ async def test_runtime_health_reports_stopped_activity_after_owned_process_exit( @pytest.mark.parametrize("error", [httpx.RemoteProtocolError, httpx.ReadTimeout]) @pytest.mark.parametrize("changed_context", [False, True]) +@pytest.mark.parametrize("exited", [False, True]) async def test_runtime_health_preserves_process_exit_during_transport_failure( - runtime_health, error, changed_context + runtime_health, error, changed_context, exited ): runtime, _, _ = runtime_health runtime._process = MagicMock(spec=asyncio.subprocess.Process) runtime._process.returncode = None async def exit_during_request(request): - runtime._process.returncode = 1 + if exited: + runtime._process.returncode = 1 if changed_context: runtime._context = None raise error("TEST_ONLY_PRIVATE", request=request) @@ -226,11 +228,18 @@ async def exit_during_request(request): assert result["reason_code"] == "observation_changed" assert result["activity"] == "unknown" assert result["observed_at"] is None - else: + elif exited: assert result["outcome"] == "failed" assert result["reason_code"] == "agent_exited" assert result["activity"] == "stopped" assert result["observed_at"] is not None + else: + assert result["outcome"] == "unknown" + assert result["reason_code"] == ( + "probe_timeout" if error is httpx.ReadTimeout else "probe_error" + ) + assert result["activity"] == "unknown" + assert result["observed_at"] is None assert "TEST_ONLY_PRIVATE" not in json.dumps(result) diff --git a/tests/adapters/test_openhands.py b/tests/adapters/test_openhands.py index a45eceab3..d0a293874 100644 --- a/tests/adapters/test_openhands.py +++ b/tests/adapters/test_openhands.py @@ -898,12 +898,13 @@ async def test_health_uses_native_conversation_lifecycle( @pytest.mark.parametrize("flag", ["_cleanup_initiated", "_cleanup_complete"]) @pytest.mark.parametrize("level", ["live", "active"]) -async def test_health_detects_native_closure(flag, level): +@pytest.mark.parametrize("initialized", [False, True]) +async def test_health_detects_native_closure(flag, level, initialized): runtime = adapter.OpenHandsRuntime() conversation = MagicMock() conversation._cleanup_initiated = False conversation._cleanup_complete = False - conversation._agent_ready = True + conversation._agent_ready = initialized conversation._arun_task = None setattr(conversation, flag, True) runtime._conversation = conversation @@ -911,6 +912,7 @@ async def test_health_detects_native_closure(flag, level): result = await runtime.check(level) assert (result["outcome"], result["reason_code"]) == ("failed", "session_closed") assert result["activity"] == "stopped" + assert result["observed_at"] is not None async def test_health_rejects_unqualified_native_shape_and_missing_runtime(): diff --git a/tests/e2e/test_hermes_checks.py b/tests/e2e/test_hermes_checks.py index 7424a16f1..4dcca7f0e 100644 --- a/tests/e2e/test_hermes_checks.py +++ b/tests/e2e/test_hermes_checks.py @@ -5,6 +5,8 @@ import asyncio import os +from pathlib import Path +import signal import sys import threading @@ -239,3 +241,68 @@ async def test_hermes_native_session_health_and_closure(health_model, tmp_path): assert requests == startup_requests finally: await native.stop() + + +@pytest.mark.skipif( + sys.platform not in {"linux", "darwin"}, + reason="requires native process identity reads", +) +async def test_hermes_host_exit_does_not_hide_behind_a_responsive_runtime_server( + health_model, tmp_path, server_sdk_bridge +): + from nemo_fabric.runtime_server import transport + + endpoint, _, release, _ = health_model + release.set() + os.environ["ADAPTER_PYTHON"] = sys.executable + host, _ = server_sdk_bridge + config = _config(endpoint, tmp_path) + started = await host.handle( + { + "interface_version": 2, + "operation": "start", + "agent": config.metadata.name, + "config": config.model_dump(mode="json"), + } + ) + assert started["status"] == "succeeded", started + runtime_id = started["result"]["runtime_id"] + runtime = host.runtimes[runtime_id].runtime + target = {"agent": config.metadata.name, "runtime_id": runtime_id} + socket_path = Path(os.environ["FABRIC_HOST_SOCKET"]) + try: + # An explicit local invocation supplies the exact owned host PID. + result = await runtime.invoke(input="Return finished.") + assert result.status == "succeeded", result.to_mapping() + os.kill(result.metadata["host_pid"], signal.SIGKILL) + async with asyncio.timeout(3): + while True: + checked = await transport.client( + { + "interface_version": 2, + "operation": "check", + "level": "ready", + **target, + }, + socket_path=socket_path, + ) + if checked["result"]["outcome"] == "failed": + break + await asyncio.sleep(0.01) + report = checked["result"] + assert checked["status"] == "failed" + assert report["reason_code"] == "agent_exited" + assert report["runtime"] == {"state": "failed", "activity": "stopped"} + assert [check["name"] for check in report["checks"]] == ["live"] + assert report["checks"][0]["source"] == "fabric.adapter.liveness" + assert report["checks"][0]["observed_at"] is not None + server = await transport.client( + {"interface_version": 2, "operation": "check"}, socket_path=socket_path + ) + assert server["result"]["outcome"] == "passed" + assert host.runtimes[runtime_id].runtime is runtime + finally: + stopped = await host.handle( + {"interface_version": 2, "operation": "stop", **target} + ) + assert stopped["status"] == "succeeded", stopped From 4efe9f32bfa2202d22a05a286ca71e325d556ead Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Wed, 4 Nov 2026 20:20:09 -0700 Subject: [PATCH 2/5] fix: preserve host exit evidence through registry contention Signed-off-by: Sara Tadayon --- crates/fabric-core/src/runtime.rs | 76 ++++++++++++++++++++++++++++--- docs/sdk/health-checks.mdx | 2 +- 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 271128120..f2ee0948a 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -2629,12 +2629,10 @@ fn local_host_processes() -> std::sync::MutexGuard<'static, BTreeMap Option { - let process = LOCAL_HOST_PROCESSES - .try_lock() - .ok()? - .get(runtime_id) - .cloned()?; + let process = local_host_processes().get(runtime_id).cloned()?; process.has_exited() } @@ -6582,6 +6580,55 @@ for line in sys.stdin: let _ = fs::remove_dir_all(root); } + #[test] + #[cfg(any(target_os = "linux", target_os = "macos"))] + fn local_host_exit_evidence_survives_registry_contention_and_poisoning() { + use std::sync::mpsc; + + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + { + let mut host = host.lock().unwrap(); + host.child.kill().unwrap(); + host.child.wait().unwrap(); + } + + let runtime_id = runtime.runtime_id.clone(); + let (entered_tx, entered_rx) = mpsc::channel(); + let (observed_tx, observed_rx) = mpsc::channel(); + let processes = local_host_processes(); + let observer = thread::spawn(move || { + entered_tx.send(()).unwrap(); + observed_tx + .send(local_host_has_exited(&runtime_id)) + .unwrap(); + }); + entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let early = observed_rx.recv_timeout(Duration::from_millis(50)); + drop(processes); + let contended = match early { + Ok(observation) => observation, + Err(mpsc::RecvTimeoutError::Timeout) => { + observed_rx.recv_timeout(Duration::from_secs(2)).unwrap() + } + Err(error) => panic!("receive exit observation: {error}"), + }; + observer.join().unwrap(); + + let poisoning = std::panic::catch_unwind(|| { + let _processes = local_host_processes(); + panic!("poison the process registry for this test"); + }); + let poisoned = local_host_has_exited(&runtime.runtime_id); + LOCAL_HOST_PROCESSES.clear_poison(); + + stop_runtime(&plan, &runtime).unwrap(); + fs::remove_dir_all(root).unwrap(); + assert!(poisoning.is_err()); + assert_eq!((contended, poisoned), (Some(true), Some(true))); + } + #[test] #[cfg(any(target_os = "linux", target_os = "macos"))] fn health_preserves_a_host_exit_during_a_failed_probe() { @@ -6613,7 +6660,24 @@ for line in sys.stdin: .unwrap(); fs::set_permissions(&endpoint, fs::Permissions::from_mode(0o600)).unwrap(); let exited = thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); + listener.set_nonblocking(true).unwrap(); + let accept_deadline = Instant::now() + Duration::from_secs(5); + let (mut stream, _) = loop { + match listener.accept() { + Ok(connection) => break connection, + Err(error) + if error.kind() == ErrorKind::WouldBlock + && Instant::now() < accept_deadline => + { + thread::sleep(Duration::from_millis(10)); + } + Err(error) if error.kind() == ErrorKind::WouldBlock => { + panic!("timed out waiting for health connection") + } + Err(error) => panic!("accept health connection: {error}"), + } + }; + stream.set_nonblocking(false).unwrap(); stream .set_read_timeout(Some(Duration::from_secs(2))) .unwrap(); diff --git a/docs/sdk/health-checks.mdx b/docs/sdk/health-checks.mdx index 0c1f60c6e..12688f314 100644 --- a/docs/sdk/health-checks.mdx +++ b/docs/sdk/health-checks.mdx @@ -81,7 +81,7 @@ mini-SWE-agent, NOOA InteractiveAgent, NOOA BenchAgent, and Remote Agent adverti Pi readiness checks configured model, tool, and MCP state. Those observations do not perform a model request or execute a tool. OpenClaw readiness follows the Gateway's channel policies and can use the Gateway's cached observations. The linked OpenClaw endpoint tests use controlled Gateway HTTP responses. -On Linux and macOS, confirmed exit of a Fabric-owned adapter host returns `failed/agent_exited` for any adapter that advertises health checks. The managed runtime can no longer serve requests, even if a separate native child process or attached service survives. This observation does not establish descendant cleanup. An unreadable process identity or a failed health connection alone remains `unknown`. +On Linux and macOS, confirmed exit of an adapter host owned by NeMo Fabric returns `failed/agent_exited` for any adapter that advertises health checks. The managed runtime can no longer serve requests, even if a separate native child process or attached service survives. This observation does not establish descendant cleanup. An unreadable process identity or a failed health connection alone remains `unknown`. Adapter checks also preserve confirmed native-process exit and closed-session evidence when a native health request fails. A changed runtime or session identity still invalidates the observation and returns `unknown/observation_changed`. From b9364e6f06b34e9f34844caf4cb16f182c748d52 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Wed, 4 Nov 2026 21:06:38 -0700 Subject: [PATCH 3/5] docs: clarify health check ordering and Kilo session identity Signed-off-by: Sara Tadayon --- docs/sdk/health-checks.mdx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/sdk/health-checks.mdx b/docs/sdk/health-checks.mdx index 12688f314..d3339a0a8 100644 --- a/docs/sdk/health-checks.mdx +++ b/docs/sdk/health-checks.mdx @@ -81,9 +81,11 @@ mini-SWE-agent, NOOA InteractiveAgent, NOOA BenchAgent, and Remote Agent adverti Pi readiness checks configured model, tool, and MCP state. Those observations do not perform a model request or execute a tool. OpenClaw readiness follows the Gateway's channel policies and can use the Gateway's cached observations. The linked OpenClaw endpoint tests use controlled Gateway HTTP responses. -On Linux and macOS, confirmed exit of an adapter host owned by NeMo Fabric returns `failed/agent_exited` for any adapter that advertises health checks. The managed runtime can no longer serve requests, even if a separate native child process or attached service survives. This observation does not establish descendant cleanup. An unreadable process identity or a failed health connection alone remains `unknown`. +On Linux and macOS, confirmed exit of an adapter host owned by NeMo Fabric returns `failed/agent_exited` at `live` for any adapter that advertises `live`. Checks evaluate `live` before higher levels, so a confirmed host exit fails a `ready` check even when the adapter does not support `ready`. The managed runtime can no longer serve requests, even if a separate native child process or attached service survives. This observation does not establish descendant cleanup. An unreadable process identity or a failed health connection alone remains `unknown`. -Adapter checks also preserve confirmed native-process exit and closed-session evidence when a native health request fails. A changed runtime or session identity still invalidates the observation and returns `unknown/observation_changed`. +Adapter checks also preserve confirmed native-process exit and closed-session evidence when a native health request fails. A changed runtime or NeMo Fabric session invalidates the observation and returns `unknown/observation_changed`. + +Kilo checks the NeMo Fabric session and runtime generation at every level. The `active` probe also returns `unknown/observation_changed` when the native session identifier differs from the selected session. Kilo's `ready` probe checks server health without comparing the native session identifier; a full `ready` check still runs `active` first. Passing a health check does not establish successful inference, valid model credentials, provider availability, or successful tool execution unless the particular probe verifies that property. For example, Claude's readiness response does not require connected MCP servers, and Qwen's command response does not test model access. The linked qualification tests hold a local model response to verify that health checks can complete during invocation; those tests provide separate invocation evidence rather than changing what the health result means. From f40bc84fd18f508dad4a0644b4305832ff4e843c Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 5 Nov 2026 09:11:24 -0700 Subject: [PATCH 4/5] fix: preserve Kilo session changes during process exit Signed-off-by: Sara Tadayon --- adapters/typescript/kilo/src/kilo-sdk.ts | 8 +++++ .../typescript/kilo/test/health-sdk.test.mjs | 32 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/adapters/typescript/kilo/src/kilo-sdk.ts b/adapters/typescript/kilo/src/kilo-sdk.ts index c3c5ae9f5..d4342ab44 100644 --- a/adapters/typescript/kilo/src/kilo-sdk.ts +++ b/adapters/typescript/kilo/src/kilo-sdk.ts @@ -477,6 +477,14 @@ class KiloSdkSessionHandle implements KiloSessionHandle { ) : await this.client.global.health(options); if (this.stopped) return unobservedHealth("observation_changed"); + if ( + level === "active" && + isRecord(response.data) && + typeof response.data.id === "string" && + response.data.id.length > 0 && + response.data.id !== this.id + ) + return unobservedHealth("observation_changed"); if (!this.server.isAlive()) return exited(); if (!isRecord(response.data)) return unobservedHealth("probe_error"); if (level === "active" && response.data.id !== this.id) diff --git a/adapters/typescript/kilo/test/health-sdk.test.mjs b/adapters/typescript/kilo/test/health-sdk.test.mjs index c63a3673b..1313f8889 100644 --- a/adapters/typescript/kilo/test/health-sdk.test.mjs +++ b/adapters/typescript/kilo/test/health-sdk.test.mjs @@ -147,6 +147,38 @@ test("native query failure remains unknown and never leaks errors", async (t) => assert.equal(state.closed, 0); }); +for (const alive of [true, false]) { + test(`a changed native session invalidates an active check when the server is ${alive ? "alive" : "exited"}`, async (t) => { + const { handle, state, client } = await session(t); + client.session.get = async () => { + state.alive = alive; + return { data: { id: "PRIVATE-replacement-session" } }; + }; + const result = await handle.check("active"); + assert.equal(result.outcome, "unknown"); + assert.equal(result.reason_code, "observation_changed"); + assert.equal(JSON.stringify(result).includes("PRIVATE"), false); + assert.equal(handle.id, "session-1"); + assert.equal(state.aborts, 0); + assert.equal(state.closed, 0); + }); +} + +for (const data of [null, {}, { id: null }, { id: 123 }, { id: "" }]) { + test(`a malformed native session response ${JSON.stringify(data)} preserves server exit evidence`, async (t) => { + const { handle, state, client } = await session(t); + client.session.get = async () => { + state.alive = false; + return { data }; + }; + const result = await handle.check("active"); + assert.equal(result.outcome, "failed"); + assert.equal(result.reason_code, "agent_exited"); + assert.equal(state.aborts, 0); + assert.equal(state.closed, 0); + }); +} + test("native health request deadlines do not abort or close an invocation", async (t) => { const keepAlive = setInterval(() => {}, 1000); t.after(() => clearInterval(keepAlive)); From abe0e56012953f25d65ed2bb34fcaf61a25d5be7 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 5 Nov 2026 21:24:14 -0700 Subject: [PATCH 5/5] fix: retain host exit evidence after invocation timeouts Signed-off-by: Sara Tadayon --- crates/fabric-core/src/runtime.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index f2ee0948a..21ee30ae6 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -2467,7 +2467,6 @@ fn invalidate_timed_out_local_host( .is_some_and(|host| Arc::ptr_eq(host, expected_host)) { hosts.remove(runtime_id); - local_host_processes().remove(runtime_id); } } @@ -5776,6 +5775,7 @@ for line in sys.stdin: fn local_host_invoke_timeout_evicts_and_terminates_host() { let (root, plan) = local_host_plan("invoke_timeout"); let runtime = start_runtime(&plan).expect("start local host"); + crate::check::register(&runtime, vec![crate::check::CheckLevel::Live]); let host = local_hosts() .get(&runtime.runtime_id) .cloned() @@ -5810,6 +5810,25 @@ for line in sys.stdin: ); assert!(!runtime_dir.exists()); + #[cfg(any(target_os = "linux", target_os = "macos"))] + for level in ["live", "active", "ready"] { + let request = serde_json::from_value(serde_json::json!({ + "scope": "runtime", "runtime_id": runtime.runtime_id, + "level": level, "timeout_seconds": 0.5, + })) + .unwrap(); + let report = crate::check::check_environment(&runtime.environment, &request).unwrap(); + assert_eq!(report.outcome, crate::check::CheckOutcome::Failed); + assert_eq!( + report.reason_code, + Some(crate::check::CheckReason::AgentExited) + ); + assert_eq!( + report.runtime.unwrap().state, + crate::check::CheckRuntimeState::Failed + ); + } + let retry = invoke_runtime(&plan, &runtime, RunRequest::text("retry")) .expect_err("timed-out runtime must remain unavailable"); assert!(matches!( @@ -5818,6 +5837,7 @@ for line in sys.stdin: )); let stopped = stop_runtime(&plan, &runtime).expect("stop after timeout is idempotent"); assert_eq!(stopped[0].metadata["already_stopped"], true); + assert!(local_host_has_exited(&runtime.runtime_id).is_none()); let _ = fs::remove_dir_all(root); }