From 34ba1cf293b37f4ad6c9a164c47d07cce5abe91c Mon Sep 17 00:00:00 2001 From: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:37:07 +0530 Subject: [PATCH 1/7] fix(sandbox): recover managed gateway after start Signed-off-by: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> --- src/lib/actions/sandbox/connect.ts | 17 ++- src/lib/actions/sandbox/start.test.ts | 149 ++++++++++++++++++++++++-- src/lib/actions/sandbox/start.ts | 72 +++++++++++-- 3 files changed, 220 insertions(+), 18 deletions(-) diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 81a92fdc23f..fd4d66980a1 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -91,6 +91,8 @@ import { applyOpenShellVmDnsMonkeypatch, shouldApplyVmDnsMonkeypatch } from "./v export { runConnectAutoPairApprovalPass }; +export type SandboxStartupRecoveryResult = ReturnType; + export type SandboxConnectOptions = { probeOnly?: boolean; }; @@ -925,8 +927,19 @@ function maybeEnsureHermesToolGatewayBroker(sb: SandboxEntry | null): void { } } -export function restoreSandboxStartupState(sandboxName: string): void { - checkAndRecoverSandboxProcesses(sandboxName, { quiet: true }); +export function restoreSandboxStartupState( + sandboxName: string, + { requireManagedRecovery = false }: { requireManagedRecovery?: boolean } = {}, +): SandboxStartupRecoveryResult { + return checkAndRecoverSandboxProcesses(sandboxName, { + quiet: true, + // `start` has already brought the existing container runtime back. For a + // built-in gateway, enter the authenticated managed recovery path even + // when the first sandbox-exec health probe is unavailable while OpenShell + // is still publishing the restarted container. The controller remains the + // authority for already-running, recovery, integrity, and refusal results. + ...(requireManagedRecovery ? { isSandboxGatewayRunningImpl: () => false } : {}), + }); } function restoreInteractiveTerminal(): void { diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index e6ebea0d12b..70bd70dde0e 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -10,7 +10,19 @@ import { } from "../../onboard/runtime-provider/docker"; import { createRuntimeProviderBundleRegistry } from "../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../state/registry"; -import { restoreStoppedSandboxStartupState, type SandboxStartDeps, startSandbox } from "./start"; +import { + assertSandboxStartupRecovery, + restoreStoppedSandboxStartupState, + type SandboxStartDeps, + startSandbox, +} from "./start"; + +const RUNNING_RECOVERY = { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, +} as const; function sandbox(values: Partial = {}): SandboxEntry { return { name: "my-sandbox", ...values }; @@ -45,7 +57,9 @@ function harness(overrides: Partial = {}) { const verifyGateway = vi.fn>(() => Promise.resolve(), ); - const restoreStartupState = vi.fn>(); + const restoreStartupState = vi.fn>( + () => RUNNING_RECOVERY, + ); const log = vi.fn<(message: string) => void>(); const runtimeProviders = createRuntimeProviderBundleRegistry([ [ @@ -85,16 +99,17 @@ function harness(overrides: Partial = {}) { describe("startSandbox", () => { it("restores sealed access before recovering sandbox processes (#8112)", () => { const restoreAccess = vi.fn(); - const restoreProcesses = vi.fn(); + const restoreProcesses = vi.fn(() => RUNNING_RECOVERY); - restoreStoppedSandboxStartupState("my-sandbox", { + const result = restoreStoppedSandboxStartupState("my-sandbox", { agent: "openclaw", restoreLockedStartupAccess: restoreAccess, restoreProcessState: restoreProcesses, }); + expect(result).toBe(RUNNING_RECOVERY); expect(restoreAccess).toHaveBeenCalledWith("my-sandbox"); - expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox"); + expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox", true); expect(restoreAccess.mock.invocationCallOrder[0]).toBeLessThan( restoreProcesses.mock.invocationCallOrder[0], ); @@ -102,16 +117,28 @@ describe("startSandbox", () => { it("keeps Hermes sealed state untouched while recovering sandbox processes (#8112)", () => { const restoreAccess = vi.fn(); - const restoreProcesses = vi.fn(); + const restoreProcesses = vi.fn(() => RUNNING_RECOVERY); - restoreStoppedSandboxStartupState("my-sandbox", { + const result = restoreStoppedSandboxStartupState("my-sandbox", { agent: "hermes", restoreLockedStartupAccess: restoreAccess, restoreProcessState: restoreProcesses, }); + expect(result).toBe(RUNNING_RECOVERY); expect(restoreAccess).not.toHaveBeenCalled(); - expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox"); + expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox", true); + }); + + it("does not force managed recovery for custom agents (#8662)", () => { + const restoreProcesses = vi.fn(() => RUNNING_RECOVERY); + + restoreStoppedSandboxStartupState("my-sandbox", { + agent: "custom-agent", + restoreProcessState: restoreProcesses, + }); + + expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox", false); }); it("restores startup state before probing readiness after a stopped container starts (#8112)", async () => { @@ -152,6 +179,112 @@ describe("startSandbox", () => { ); }); + it("fails before readiness when the managed startup result is inconclusive (#8662)", async () => { + const h = harness(); + h.restoreStartupState.mockReturnValue({ + checked: false, + wasRunning: null, + recovered: false, + forwardRecovered: false, + }); + + await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow( + "managed gateway could not be inspected through the trusted container boundary", + ); + expect(h.verifyGateway).not.toHaveBeenCalled(); + }); + + it("verifies readiness after managed gateway recovery succeeds (#8662)", async () => { + const h = harness(); + h.restoreStartupState.mockReturnValue({ + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: true, + }); + + await expect(startSandbox("my-sandbox", h.deps)).resolves.toEqual({ exitCode: 0 }); + expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); + expect(h.restoreStartupState.mock.invocationCallOrder[0]).toBeLessThan( + h.verifyGateway.mock.invocationCallOrder[0], + ); + }); + + it.each([ + [ + "secret-boundary refusal", + { + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "raw-secret", + }, + "secret-boundary check refused", + ], + [ + "MCP reconciliation refusal", + { + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + mcpReconciliationRefused: true, + mcpReconciliationReason: "raw diagnostic remains private", + }, + "MCP configuration does not match", + ], + [ + "forward recovery failure", + { + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: "raw diagnostic remains private", + }, + "required host forwards could not be restored", + ], + [ + "managed recovery failure", + { + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }, + "authenticated managed gateway recovery did not complete", + ], + ] as const)("fails closed on %s without exposing raw recovery details (#8662)", async (_label, recovery, expected) => { + const h = harness(); + h.restoreStartupState.mockReturnValue(recovery); + + let failure: unknown; + try { + await startSandbox("my-sandbox", h.deps); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain(expected); + expect((failure as Error).message).not.toContain("raw diagnostic remains private"); + expect(h.verifyGateway).not.toHaveBeenCalled(); + }); + + it("accepts terminal agents without requiring gateway recovery (#8662)", () => { + expect(() => + assertSandboxStartupRecovery("terminal-box", { + checked: true, + wasRunning: null, + recovered: false, + forwardRecovered: false, + runtime: "terminal", + }), + ).not.toThrow(); + }); + it("reports the started container by name (#6026)", async () => { const h = harness(); diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 7e904befd0a..ce245c35987 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -1,12 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { CLI_NAME } from "../../cli/branding"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, type RuntimeProviderBundleRegistry, } from "../../onboard/runtime-provider/access"; import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; +import type { SandboxStartupRecoveryResult } from "./connect"; import { resolveSandboxLifecycleProvider, type SandboxLifecycleResult, @@ -19,9 +21,12 @@ function verifyGateway(sandboxName: string): Promise { return connectSandbox(sandboxName, { probeOnly: true }); } -function restoreProcessState(sandboxName: string): void { +function restoreProcessState( + sandboxName: string, + requireManagedRecovery: boolean, +): SandboxStartupRecoveryResult { const { restoreSandboxStartupState } = require("./connect") as typeof import("./connect"); - restoreSandboxStartupState(sandboxName); + return restoreSandboxStartupState(sandboxName, { requireManagedRecovery }); } function restoreLockedStartupAccess(sandboxName: string): void { @@ -33,24 +38,74 @@ function restoreLockedStartupAccess(sandboxName: string): void { export interface SandboxStartupStateDeps { agent?: SandboxEntry["agent"]; restoreLockedStartupAccess?: (sandboxName: string) => void; - restoreProcessState?: (sandboxName: string) => void; + restoreProcessState?: typeof restoreProcessState; } export function restoreStoppedSandboxStartupState( sandboxName: string, deps: SandboxStartupStateDeps = {}, -): void { - if ((deps.agent ?? "openclaw") === "openclaw") { +): SandboxStartupRecoveryResult { + const agent = deps.agent ?? "openclaw"; + if (agent === "openclaw") { (deps.restoreLockedStartupAccess ?? restoreLockedStartupAccess)(sandboxName); } - (deps.restoreProcessState ?? restoreProcessState)(sandboxName); + return (deps.restoreProcessState ?? restoreProcessState)( + sandboxName, + agent === "openclaw" || agent === "hermes", + ); +} + +function startupRecoveryFailure( + sandboxName: string, + detail: string, + nextStep = `Run '${CLI_NAME} ${sandboxName} recover' for classified repair guidance.`, +): Error { + return new Error(`Could not restore sandbox '${sandboxName}' after start: ${detail} ${nextStep}`); +} + +export function assertSandboxStartupRecovery( + sandboxName: string, + recovery: SandboxStartupRecoveryResult, +): void { + if ("runtime" in recovery && recovery.runtime === "terminal") return; + if (!recovery.checked) { + throw startupRecoveryFailure( + sandboxName, + "the managed gateway could not be inspected through the trusted container boundary.", + ); + } + if ("secretBoundaryRefused" in recovery && recovery.secretBoundaryRefused) { + throw startupRecoveryFailure( + sandboxName, + "the Hermes secret-boundary check refused the gateway state; Shields were left unchanged.", + ); + } + if ("mcpReconciliationRefused" in recovery && recovery.mcpReconciliationRefused) { + throw startupRecoveryFailure( + sandboxName, + "the Hermes MCP configuration does not match its persisted managed intent; Shields were left unchanged.", + ); + } + if ("forwardRecoveryFailed" in recovery && recovery.forwardRecoveryFailed) { + throw startupRecoveryFailure( + sandboxName, + "one or more required host forwards could not be restored.", + ); + } + if (!recovery.wasRunning && !recovery.recovered) { + throw startupRecoveryFailure( + sandboxName, + "authenticated managed gateway recovery did not complete.", + `Inspect '${CLI_NAME} ${sandboxName} logs', then run '${CLI_NAME} ${sandboxName} recover'.`, + ); + } } export interface SandboxStartDeps { environment?: NodeJS.ProcessEnv; getSandbox?: typeof registry.getSandbox; runtimeProviders?: RuntimeProviderBundleRegistry; - restoreStartupState?: (sandboxName: string) => void; + restoreStartupState?: (sandboxName: string) => SandboxStartupRecoveryResult; verifyGateway?: (sandboxName: string) => Promise; log?: (message: string) => void; } @@ -93,7 +148,8 @@ export async function startSandbox( restoreStoppedSandboxStartupState(sandboxNameToRestore, { agent: resolved.sandbox.agent, })); - restoreStartupState(name); + const recovery = restoreStartupState(name); + assertSandboxStartupRecovery(name, recovery); log(" Checking gateway health and host forwards…"); await (deps.verifyGateway ?? verifyGateway)(name); }); From 95c6fd8ea2946388ebec06aac981a80222933e67 Mon Sep 17 00:00:00 2001 From: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:51:26 +0530 Subject: [PATCH 2/7] fix(sandbox): preserve custom-agent startup Signed-off-by: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> --- src/lib/actions/sandbox/start.test.ts | 32 ++++++++++++++++++++++----- src/lib/actions/sandbox/start.ts | 11 +++++++-- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index 70bd70dde0e..de852f15df0 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -23,6 +23,8 @@ const RUNNING_RECOVERY = { recovered: false, forwardRecovered: false, } as const; +const SECRET_BOUNDARY_PRIVATE_DETAIL = "raw-secret" as const; +const PRIVATE_RECOVERY_DETAIL = "raw diagnostic remains private"; function sandbox(values: Partial = {}): SandboxEntry { return { name: "my-sandbox", ...values }; @@ -194,6 +196,20 @@ describe("startSandbox", () => { expect(h.verifyGateway).not.toHaveBeenCalled(); }); + it("keeps custom-agent readiness authoritative when recovery is inconclusive (#8662)", async () => { + const h = harness(); + h.getSandbox.mockReturnValue(sandbox({ agent: "custom-agent" })); + h.restoreStartupState.mockReturnValue({ + checked: false, + wasRunning: null, + recovered: false, + forwardRecovered: false, + }); + + await expect(startSandbox("my-sandbox", h.deps)).resolves.toEqual({ exitCode: 0 }); + expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); + }); + it("verifies readiness after managed gateway recovery succeeds (#8662)", async () => { const h = harness(); h.restoreStartupState.mockReturnValue({ @@ -219,9 +235,10 @@ describe("startSandbox", () => { recovered: false, forwardRecovered: false, secretBoundaryRefused: true, - secretBoundaryReason: "raw-secret", + secretBoundaryReason: SECRET_BOUNDARY_PRIVATE_DETAIL, }, "secret-boundary check refused", + SECRET_BOUNDARY_PRIVATE_DETAIL, ], [ "MCP reconciliation refusal", @@ -231,9 +248,10 @@ describe("startSandbox", () => { recovered: false, forwardRecovered: false, mcpReconciliationRefused: true, - mcpReconciliationReason: "raw diagnostic remains private", + mcpReconciliationReason: PRIVATE_RECOVERY_DETAIL, }, "MCP configuration does not match", + PRIVATE_RECOVERY_DETAIL, ], [ "forward recovery failure", @@ -243,9 +261,10 @@ describe("startSandbox", () => { recovered: true, forwardRecovered: false, forwardRecoveryFailed: true, - forwardRecoveryFailureDetail: "raw diagnostic remains private", + forwardRecoveryFailureDetail: PRIVATE_RECOVERY_DETAIL, }, "required host forwards could not be restored", + PRIVATE_RECOVERY_DETAIL, ], [ "managed recovery failure", @@ -256,8 +275,9 @@ describe("startSandbox", () => { forwardRecovered: false, }, "authenticated managed gateway recovery did not complete", + undefined, ], - ] as const)("fails closed on %s without exposing raw recovery details (#8662)", async (_label, recovery, expected) => { + ] as const)("fails closed on %s without exposing raw recovery details (#8662)", async (_label, recovery, expected, privateDetail) => { const h = harness(); h.restoreStartupState.mockReturnValue(recovery); @@ -269,7 +289,9 @@ describe("startSandbox", () => { } expect(failure).toBeInstanceOf(Error); expect((failure as Error).message).toContain(expected); - expect((failure as Error).message).not.toContain("raw diagnostic remains private"); + if (privateDetail) { + expect((failure as Error).message).not.toContain(privateDetail); + } expect(h.verifyGateway).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index ce245c35987..49475fce8fc 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -41,6 +41,11 @@ export interface SandboxStartupStateDeps { restoreProcessState?: typeof restoreProcessState; } +function requiresManagedStartupRecovery(agent: SandboxEntry["agent"]): boolean { + const resolvedAgent = agent ?? "openclaw"; + return resolvedAgent === "openclaw" || resolvedAgent === "hermes"; +} + export function restoreStoppedSandboxStartupState( sandboxName: string, deps: SandboxStartupStateDeps = {}, @@ -51,7 +56,7 @@ export function restoreStoppedSandboxStartupState( } return (deps.restoreProcessState ?? restoreProcessState)( sandboxName, - agent === "openclaw" || agent === "hermes", + requiresManagedStartupRecovery(agent), ); } @@ -149,7 +154,9 @@ export async function startSandbox( agent: resolved.sandbox.agent, })); const recovery = restoreStartupState(name); - assertSandboxStartupRecovery(name, recovery); + if (requiresManagedStartupRecovery(resolved.sandbox.agent)) { + assertSandboxStartupRecovery(name, recovery); + } log(" Checking gateway health and host forwards…"); await (deps.verifyGateway ?? verifyGateway)(name); }); From 5e81fe908e640c31d24fe4b66c73e22600dd60ad Mon Sep 17 00:00:00 2001 From: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:57:17 +0530 Subject: [PATCH 3/7] test(sandbox): keep recovery cases linear Signed-off-by: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> --- src/lib/actions/sandbox/start.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index de852f15df0..08264353c3c 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -275,7 +275,7 @@ describe("startSandbox", () => { forwardRecovered: false, }, "authenticated managed gateway recovery did not complete", - undefined, + PRIVATE_RECOVERY_DETAIL, ], ] as const)("fails closed on %s without exposing raw recovery details (#8662)", async (_label, recovery, expected, privateDetail) => { const h = harness(); @@ -289,9 +289,7 @@ describe("startSandbox", () => { } expect(failure).toBeInstanceOf(Error); expect((failure as Error).message).toContain(expected); - if (privateDetail) { - expect((failure as Error).message).not.toContain(privateDetail); - } + expect((failure as Error).message).not.toContain(privateDetail); expect(h.verifyGateway).not.toHaveBeenCalled(); }); From 271f26b4a69f107b3260fc276a7d7a21ee3cec64 Mon Sep 17 00:00:00 2001 From: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:10:15 +0530 Subject: [PATCH 4/7] fix(sandbox): prove protected restart recovery Co-authored-by: Apurv Kumaria Signed-off-by: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> --- .../recover-rebuild-sandboxes.mdx | 22 +- docs/manage-sandboxes/run-sandboxes.mdx | 4 +- docs/reference/commands.mdx | 23 + src/lib/actions/sandbox/connect-flow.test.ts | 69 +- src/lib/actions/sandbox/connect.ts | 103 +- src/lib/actions/sandbox/docker-health.test.ts | 8 +- src/lib/actions/sandbox/forward-recovery.ts | 64 +- .../actions/sandbox/process-recovery.test.ts | 191 +++ src/lib/actions/sandbox/process-recovery.ts | 712 +++++++++-- src/lib/actions/sandbox/start.test.ts | 532 ++++++-- src/lib/actions/sandbox/start.ts | 327 +++-- src/lib/actions/sandbox/stop.test.ts | 57 +- .../docker-driver-sandbox-recovery.test.ts | 110 +- .../onboard/docker-driver-sandbox-recovery.ts | 82 +- src/lib/onboard/runtime-provider/contract.ts | 2 + src/lib/onboard/runtime-provider/docker.ts | 44 +- .../onboard/runtime-provider/podman.test.ts | 14 +- src/lib/shields/index.ts | 47 +- src/lib/shields/startup-access.test.ts | 103 ++ test/e2e/live/hermes-shields-config.test.ts | 114 +- test/e2e/live/shields-config.test.ts | 63 +- .../live/shields-restart-recovery-evidence.ts | 1130 +++++++++++++++++ ...ocess-recovery-supervisor-relaunch.test.ts | 7 +- 23 files changed, 3357 insertions(+), 471 deletions(-) create mode 100644 src/lib/shields/startup-access.test.ts create mode 100644 test/e2e/live/shields-restart-recovery-evidence.ts diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index c0d95dc2fe9..6e9cf5f58e8 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -22,24 +22,24 @@ $$nemoclaw start ``` This path preserves the sandbox workspace and repairs the agent runtime and host-side forwards after the container starts. -If the container is paused, follow the printed `docker unpause` guidance instead. +If the Docker container is paused, `start` unpauses it. If the container is missing or OpenShell reports another terminal phase such as `Failed`, follow the printed `rebuild --yes` guidance so NemoClaw can recreate the sandbox from its recorded metadata. + +For a Docker-driver sandbox, `start` binds managed recovery and verification to the existing container's immutable runtime identity. +The command preserves the current Shields posture and does not rebuild, replace, or destroy the container. +If another recovery layer fails, run the printed `recover` command before you retry `start`. + + -If the sandbox has shields up and the OpenClaw gateway does not start after the container restarts, lower shields before you rebuild: +If an expired Shields auto-restore blocks locked startup access, finish the restore and retry `start`: ```bash -$$nemoclaw shields down +$$nemoclaw shields up +$$nemoclaw start ``` -While holding the config mutation lock, NemoClaw confirms that no startup process runs and no readiness lease exists. -Only then does it accept `shields down`. -Other locked-config operations still require the lease. -This failed-startup recovery path requires a sandbox image that includes the in-container OpenClaw config and state guards. -If NemoClaw reports that the config guard is absent, upgrade the CLI. -Then rebuild the sandbox before you retry recovery. -After shields are down, start the sandbox again. -When the OpenClaw gateway is healthy, rerun `shields up`. +Use `shields down` only when you intend to change protected configuration. ## Recover the Agent Runtime diff --git a/docs/manage-sandboxes/run-sandboxes.mdx b/docs/manage-sandboxes/run-sandboxes.mdx index d19c5eb2057..2c077554540 100644 --- a/docs/manage-sandboxes/run-sandboxes.mdx +++ b/docs/manage-sandboxes/run-sandboxes.mdx @@ -132,8 +132,10 @@ $$nemoclaw start ``` -After Docker reports the existing container as running, NemoClaw checks the selected agent's managed startup state and recovers missing processes when needed. +After the lifecycle provider reports the existing runtime as running, NemoClaw checks the selected agent's managed startup state and recovers missing processes when needed. NemoClaw completes this recovery before its final OpenShell readiness and host-forward checks. +For the Docker driver, the command binds recovery and verification to the existing container's immutable runtime identity. +It does not recreate the container. After Docker reports the existing container as running, NemoClaw verifies the managed terminal runtime before the command reports success. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 2fd0fbdf684..06ac067b975 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1538,6 +1538,29 @@ Starting an already-running sandbox skips the container start and still runs the A paused container is unpaused. If the container was removed entirely, `start` fails and points you to `$$nemoclaw rebuild`. + + +For an OpenClaw or Hermes sandbox that uses the Docker driver, `start` binds recovery and verification to the existing container's immutable runtime identity. +The command does not rebuild, replace, or destroy that container. + +`start` returns success only after these checks pass: + +- The authenticated managed gateway is healthy in the same runtime identity. +- OpenShell reports the sandbox in the `Ready` phase. +- Every required host forward is active. +- A final authenticated managed-gateway check passes after inference-route verification and any applicable best-effort pairing pass. + +If a check fails, the command exits nonzero, identifies the failed layer, and preserves the existing container. +A failure after the Docker start can leave that container running but unready. +For most recovery failures, run the printed `$$nemoclaw recover` command before you retry `start`. + + + + +If an expired Shields auto-restore blocks startup access, run `$$nemoclaw shields up`, then retry `$$nemoclaw start`. + + + ### `$$nemoclaw status` Show sandbox-scoped status, health, and inference configuration for one registered sandbox. diff --git a/src/lib/actions/sandbox/connect-flow.test.ts b/src/lib/actions/sandbox/connect-flow.test.ts index df5cea7d022..795a6929f1e 100644 --- a/src/lib/actions/sandbox/connect-flow.test.ts +++ b/src/lib/actions/sandbox/connect-flow.test.ts @@ -508,7 +508,7 @@ describe("connectSandbox flow", () => { expect(harness.checkAndRecoverSpy).toHaveBeenCalledWith( "alpha", - expect.objectContaining({ quiet: true }), + expect.objectContaining({ preserveContainer: false, quiet: true }), ); expect(harness.runAutoPairSpy).toHaveBeenCalledWith("alpha"); expect(harness.spawnSyncSpy).not.toHaveBeenCalledWith( @@ -521,6 +521,28 @@ describe("connectSandbox flow", () => { ); }); + it("preserves the existing container during a lifecycle-owned final probe (#8662)", async () => { + const harness = createConnectHarness(); + + await expect( + harness.connectSandbox("alpha", { + expectedContainerId: "a".repeat(64), + preserveContainer: true, + probeOnly: true, + }), + ).resolves.toBeUndefined(); + + expect(harness.checkAndRecoverSpy).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + expectedContainerId: "a".repeat(64), + preserveContainer: true, + quiet: true, + }), + ); + expect(harness.runAutoPairSpy).toHaveBeenCalledWith("alpha"); + }); + it("probe-only mode exits before reporting success when inference.local returns no trusted result (#8502)", async () => { const harness = createConnectHarness({ registryEntry: { @@ -580,6 +602,51 @@ describe("connectSandbox flow", () => { ); expect(exitSpy).toHaveBeenCalledWith(1); }); + + it.each([ + { + expectedLayer: "managed supervisor", + processCheck: { + checked: false, + forwardRecovered: false, + managedSupervisorFailureDetail: + "supervisor unavailable: Authorization: Bearer opaque-final-probe-token", + managedSupervisorUnavailable: true, + recovered: false, + wasRunning: false, + }, + }, + { + expectedLayer: "OpenShell readiness", + processCheck: { + checked: true, + forwardRecovered: false, + openshellReadinessFailed: true, + openshellReadinessFailureDetail: + "the sandbox did not become ready in OpenShell, so its host forward was not started", + recovered: false, + wasRunning: false, + }, + }, + ])("probe-only mode preserves the actionable $expectedLayer failure (#8662)", async ({ + expectedLayer, + processCheck, + }) => { + const harness = createConnectHarness({ processCheck }); + + await expect( + harness.connectSandbox("alpha", { preserveContainer: true, probeOnly: true }), + ).rejects.toThrow("process.exit(1)"); + + const errorOutput = harness.errorSpy.mock.calls.map((call) => String(call[0] ?? "")).join("\n"); + expect(errorOutput).toContain(expectedLayer); + expect(errorOutput).toContain("existing sandbox was preserved"); + expect(errorOutput).toContain("nemoclaw alpha recover"); + expect(errorOutput).not.toContain("opaque-final-probe-token"); + expect(errorOutput.includes("")).toBe(expectedLayer === "managed supervisor"); + expect(exitSpy).toHaveBeenCalledWith(1); + }); + it("probe-only mode reports the supported repair when relaunch is quarantined (#7801)", async () => { const harness = createConnectHarness({ processCheck: { checked: true, wasRunning: false, recovered: false }, diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index fd4d66980a1..746681c2b90 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -38,7 +38,7 @@ import { import { isWsl } from "../../platform"; import { ROOT } from "../../runner"; import * as sandboxVersion from "../../sandbox/version"; -import { redact } from "../../security/redact"; +import { redact, redactFull } from "../../security/redact"; import { isSandboxReady, isTerminalSandboxPhase, @@ -84,19 +84,31 @@ import { executeSandboxExecCommand, type GatewayRestartFailureLayer, type ManagedGatewayControlCompletion, + type ManagedGatewayRecoveryFailure, + pinnedManagedGatewayProbeFailure, resolveSandboxDashboardPort, + type SandboxProcessRecoveryOptions, } from "./process-recovery"; import { runTerminalAgentConnectProbe } from "./terminal-connect-probe"; import { applyOpenShellVmDnsMonkeypatch, shouldApplyVmDnsMonkeypatch } from "./vm-dns-monkeypatch"; -export { runConnectAutoPairApprovalPass }; - -export type SandboxStartupRecoveryResult = ReturnType; +export { pinnedManagedGatewayProbeFailure, runConnectAutoPairApprovalPass }; export type SandboxConnectOptions = { + /** Immutable direct-container identity retained by lifecycle `start`. */ + expectedContainerId?: string; probeOnly?: boolean; + /** Internal recovery guard used by `start`; this is not a CLI flag. */ + preserveContainer?: boolean; }; +export type SandboxStartupRecoveryOptions = Omit< + SandboxProcessRecoveryOptions, + "preserveContainer" | "quiet" +>; +export type SandboxStartupRecoveryResult = ReturnType; +export type SandboxStartupManagedGatewayRecoveryFailure = ManagedGatewayRecoveryFailure; + type SpawnLikeResult = { status: number | null; signal?: NodeJS.Signals | null; @@ -237,7 +249,39 @@ function exitOnForwardRecoveryFailure( process.exit(1); } -async function runSandboxConnectProbe(sandboxName: string): Promise { +function exitOnPreservedStartupRecoveryFailure( + sandboxName: string, + agentName: string, + layer: "managed gateway health" | "managed supervisor" | "OpenShell readiness", + detail: unknown, +): never { + const sanitizedDetail = sanitizeSandboxStartupRecoveryDetail( + String(detail ?? "startup recovery failed"), + ); + console.error( + ` Probe failed: ${agentName} startup recovery in '${sandboxName}' failed during ${layer}: ${sanitizedDetail}.`, + ); + console.error( + ` The existing sandbox was preserved. Run \`${CLI_NAME} ${sandboxName} recover\` and retry \`${CLI_NAME} ${sandboxName} start\`.`, + ); + process.exit(1); +} + +export function sanitizeSandboxStartupRecoveryDetail(raw: string): string { + return redactFull(raw) + .replace(/[\u0000-\u001f\u007f-\u009f]/gu, " ") + .replace(/\s+/gu, " ") + .trim() + .slice(0, 240); +} + +async function runSandboxConnectProbe( + sandboxName: string, + { + expectedContainerId, + preserveContainer = false, + }: Pick = {}, +): Promise { const agent = agentRuntime.getSessionAgent(sandboxName); const agentName = agentRuntime.getAgentDisplayName(agent); if (agent && !agentRuntime.hasGatewayRuntime(agent)) { @@ -258,10 +302,42 @@ async function runSandboxConnectProbe(sandboxName: string): Promise { let recoveryFailureLayer: GatewayRestartFailureLayer | null = null; const processCheck = checkAndRecoverSandboxProcesses(sandboxName, { quiet: true, + expectedContainerId, + preserveContainer, onRecoveryFailureLayer: (layer) => { recoveryFailureLayer = layer; }, }); + if ("managedSupervisorUnavailable" in processCheck && processCheck.managedSupervisorUnavailable) { + exitOnPreservedStartupRecoveryFailure( + sandboxName, + agentName, + "managed supervisor", + "managedSupervisorFailureDetail" in processCheck + ? processCheck.managedSupervisorFailureDetail + : "the managed supervisor did not become available", + ); + } + if ("managedGatewayProbeFailed" in processCheck && processCheck.managedGatewayProbeFailed) { + exitOnPreservedStartupRecoveryFailure( + sandboxName, + agentName, + "managed gateway health", + "managedGatewayProbeFailureDetail" in processCheck + ? processCheck.managedGatewayProbeFailureDetail + : "the authenticated managed gateway health probe did not pass", + ); + } + if ("openshellReadinessFailed" in processCheck && processCheck.openshellReadinessFailed) { + exitOnPreservedStartupRecoveryFailure( + sandboxName, + agentName, + "OpenShell readiness", + "openshellReadinessFailureDetail" in processCheck + ? processCheck.openshellReadinessFailureDetail + : "the sandbox did not become ready in OpenShell", + ); + } if (!processCheck.checked) { console.error( ` Probe failed: could not inspect the ${agentName} gateway inside sandbox '${sandboxName}'.`, @@ -929,16 +1005,12 @@ function maybeEnsureHermesToolGatewayBroker(sb: SandboxEntry | null): void { export function restoreSandboxStartupState( sandboxName: string, - { requireManagedRecovery = false }: { requireManagedRecovery?: boolean } = {}, + options: SandboxStartupRecoveryOptions = {}, ): SandboxStartupRecoveryResult { return checkAndRecoverSandboxProcesses(sandboxName, { + ...options, + preserveContainer: true, quiet: true, - // `start` has already brought the existing container runtime back. For a - // built-in gateway, enter the authenticated managed recovery path even - // when the first sandbox-exec health probe is unavailable while OpenShell - // is still publishing the restarted container. The controller remains the - // authority for already-running, recovery, integrity, and refusal results. - ...(requireManagedRecovery ? { isSandboxGatewayRunningImpl: () => false } : {}), }); } @@ -1240,7 +1312,7 @@ export async function prepareInteractiveSession( export async function connectSandbox( sandboxName: string, - { probeOnly = false }: SandboxConnectOptions = {}, + { expectedContainerId, probeOnly = false, preserveContainer = false }: SandboxConnectOptions = {}, ): Promise { if (probeOnly) { await runConnectEntryPreflight(sandboxName, { probeOnly: true }); @@ -1252,7 +1324,10 @@ export async function connectSandbox( // before any in-sandbox process or host-forward mutation. The readiness // polls are already owner-scoped; this also catches registry changes. await ensureLiveSandboxOrExit(sandboxName, { gatewayRecovery: "observe" }); - return await runSandboxConnectProbe(sandboxName); + return await runSandboxConnectProbe(sandboxName, { + expectedContainerId, + preserveContainer, + }); } const { agent, sb } = await prepareInteractiveSession(sandboxName); diff --git a/src/lib/actions/sandbox/docker-health.test.ts b/src/lib/actions/sandbox/docker-health.test.ts index 5c287788f6d..78bac356253 100644 --- a/src/lib/actions/sandbox/docker-health.test.ts +++ b/src/lib/actions/sandbox/docker-health.test.ts @@ -24,7 +24,12 @@ function fixture({ throwOnInspect?: boolean; throwOnPaused?: boolean; knownSandboxes?: string[]; - labeledContainers?: Array<{ name: string; status: string; running: boolean }>; + labeledContainers?: Array<{ + containerId: string; + name: string; + status: string; + running: boolean; + }>; } = {}) { const sandbox: Partial = { name: "my-assistant", openshellDriver: driver }; const runningNames = new Set( @@ -44,6 +49,7 @@ function fixture({ .map((name) => name.trim()) .filter(Boolean) .map((name) => ({ + containerId: "a".repeat(64), name, status: runningNames.has(name) ? "Up 1 minute" : "Exited (0) 1 minute ago", running: runningNames.has(name), diff --git a/src/lib/actions/sandbox/forward-recovery.ts b/src/lib/actions/sandbox/forward-recovery.ts index bcf4adcc366..3cc069c5683 100644 --- a/src/lib/actions/sandbox/forward-recovery.ts +++ b/src/lib/actions/sandbox/forward-recovery.ts @@ -275,6 +275,7 @@ export function ensureSandboxPortForwardForPort( const configuredWaitMs = Number(process.env.NEMOCLAW_FORWARD_RECOVERY_WAIT_MS ?? "3000"); const waitMs = Number.isFinite(configuredWaitMs) ? Math.max(0, configuredWaitMs) : 3000; + if (!beforeStart()) return false; const stopResult = runOpenshell(["forward", "stop", String(port), sandboxName], { ignoreError: true, stdio: "ignore", @@ -379,10 +380,20 @@ export function ensureSandboxPortForwardForPort( return settled && !occupied && acceptSuccessfulForward(); } -export function ensureHermesDashboardPortForwardIfEnabled(sandboxName: string): boolean | null { +export function ensureHermesDashboardPortForwardIfEnabled( + sandboxName: string, + options: SandboxForwardRecoveryOptions = {}, +): boolean | null { return ensureHermesDashboardPortForward(sandboxName, { - isPortForwardHealthy: isSandboxPortForwardHealthy, - ensurePortForward: ensureSandboxPortForwardForPort, + isPortForwardHealthy: (name, port) => { + const health = isSandboxPortForwardHealthy(name, port); + return health === true ? (options.afterSuccess?.() ?? true) : health; + }, + ensurePortForward: (name, port) => + ensureSandboxPortForwardForPort(name, port, { + afterSuccess: options.afterSuccess, + beforeStart: options.beforeStart, + }), }); } @@ -395,20 +406,33 @@ function getSandboxMessagingHostForward( return getActiveMessagingHostForward(plan); } -export function ensureMessagingHostForwardHealthy(sandboxName: string): boolean | null { +export function ensureMessagingHostForwardHealthy( + sandboxName: string, + options: SandboxForwardRecoveryOptions = {}, +): boolean | null { const forward = getSandboxMessagingHostForward(sandboxName); if (!forward) return null; const health = isSandboxPortForwardHealthy(sandboxName, forward.port); - if (health === true) return true; + if (health === true) return options.afterSuccess?.() ?? true; if (health === "occupied") return false; - return ensureSandboxPortForwardForPort(sandboxName, forward.port); + return ensureSandboxPortForwardForPort(sandboxName, forward.port, { + afterSuccess: options.afterSuccess, + beforeStart: options.beforeStart, + }); } export function recoverMessagingHostForward( sandboxName: string, - { quiet }: { quiet: boolean }, + { + afterSuccess, + beforeStart, + quiet, + }: { afterSuccess?: () => boolean; beforeStart?: () => boolean; quiet: boolean }, ): boolean | null { - const recovered = ensureMessagingHostForwardHealthy(sandboxName); + const recovered = ensureMessagingHostForwardHealthy(sandboxName, { + afterSuccess, + beforeStart, + }); if (!quiet && recovered === false) { console.error(" Messaging webhook port forward could not be re-established."); } @@ -425,6 +449,7 @@ export function recoverMessagingHostForward( export function ensureDeclaredAgentForwardPortsHealthy( sandboxName: string, primaryPort: number, + options: SandboxForwardRecoveryOptions = {}, ): boolean | null { const agent = agentRuntime.getSessionAgent(sandboxName); if (!agent) return null; @@ -443,12 +468,20 @@ export function ensureDeclaredAgentForwardPortsHealthy( if (skipSet.has(candidate)) continue; sawCovered = true; const health = isSandboxPortForwardHealthy(sandboxName, candidate); - if (health === true) continue; + if (health === true) { + if (options.afterSuccess?.() === false) allHealthy = false; + continue; + } if (health === "occupied") { allHealthy = false; continue; } - if (!ensureSandboxPortForwardForPort(sandboxName, candidate)) { + if ( + !ensureSandboxPortForwardForPort(sandboxName, candidate, { + afterSuccess: options.afterSuccess, + beforeStart: options.beforeStart, + }) + ) { allHealthy = false; } } @@ -459,9 +492,16 @@ export function ensureDeclaredAgentForwardPortsHealthy( export function recoverDeclaredAgentForwardPorts( sandboxName: string, recoveryPort: number, - { quiet }: { quiet: boolean }, + { + afterSuccess, + beforeStart, + quiet, + }: { afterSuccess?: () => boolean; beforeStart?: () => boolean; quiet: boolean }, ): boolean | null { - const recovered = ensureDeclaredAgentForwardPortsHealthy(sandboxName, recoveryPort); + const recovered = ensureDeclaredAgentForwardPortsHealthy(sandboxName, recoveryPort, { + afterSuccess, + beforeStart, + }); if (!quiet && recovered === false) { console.error(" One or more agent-declared port forwards could not be re-established."); } diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index e8b07d81201..e4c05b6ce86 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -7,6 +7,8 @@ import { parseManagedGatewayControlCompletion } from "./gateway-restart"; // Import source directly so this test cannot pass against a stale build. import { confirmRecoveredSandboxGatewayManaged, + pinnedManagedGatewayProbeFailure, + waitForManagedGatewaySupervisor, waitForRecoveredSandboxGateway, waitForRecreatedSandboxOpenShellReady, } from "./process-recovery"; @@ -71,6 +73,195 @@ describe("managed gateway control completion", () => { }); }); +describe("managed gateway supervisor startup barrier", () => { + it("pins the post-connect managed proof to the lifecycle container identity (#8662)", () => { + const expectedContainerId = "c".repeat(64); + const requestPinnedGatewaySupervisorActionImpl = vi.fn(() => ({ + status: 0, + stdout: `v1 ${"d".repeat(64)} complete already-running 4242 4242\nGATEWAY_PID=4242`, + stderr: "", + })); + const waitForManagedGatewaySupervisorImpl = vi.fn( + (sandboxName, options = {}) => { + const result = options.requestGatewaySupervisorActionImpl?.(sandboxName, "probe", 1234); + return result?.status === 0; + }, + ); + + expect( + pinnedManagedGatewayProbeFailure("restart-box", expectedContainerId, { + requestPinnedGatewaySupervisorActionImpl, + waitForManagedGatewaySupervisorImpl, + }), + ).toBeNull(); + + expect(waitForManagedGatewaySupervisorImpl).toHaveBeenCalledWith( + "restart-box", + expect.objectContaining({ acceptGatewayHealthPending: false }), + ); + expect(requestPinnedGatewaySupervisorActionImpl).toHaveBeenCalledExactlyOnceWith( + "restart-box", + "probe", + 1234, + expectedContainerId, + ); + }); + + it("classifies a final pinned identity exception without exposing its detail (#8662)", () => { + const requestPinnedGatewaySupervisorActionImpl = vi.fn(() => { + throw new Error("container identity changed: Authorization: Bearer opaque-token"); + }); + + const failure = pinnedManagedGatewayProbeFailure("restart-box", "c".repeat(64), { + requestPinnedGatewaySupervisorActionImpl, + waitForManagedGatewaySupervisorImpl: (sandboxName, options = {}) => { + options.requestGatewaySupervisorActionImpl?.(sandboxName, "probe", 1234); + return false; + }, + }); + + expect(failure).toEqual({ + layer: "privileged control unavailable", + detail: "the pinned runtime identity changed during final managed gateway verification", + }); + expect(JSON.stringify(failure)).not.toContain("opaque-token"); + }); + + it("admits only the exact health-pending result for stopped-container recovery (#8662)", () => { + const requestGatewaySupervisorActionImpl = vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "GATEWAY_HEALTH_TIMEOUT", + })); + const sleepImpl = vi.fn(); + const onFailureResult = vi.fn(); + + expect( + waitForManagedGatewaySupervisor("restart-box", { + acceptGatewayHealthPending: true, + onFailureResult, + requestGatewaySupervisorActionImpl, + sleepImpl, + }), + ).toBe(true); + + expect(requestGatewaySupervisorActionImpl).toHaveBeenCalledTimes(1); + expect(requestGatewaySupervisorActionImpl).toHaveBeenCalledWith( + "restart-box", + "probe", + expect.any(Number), + ); + expect(sleepImpl).not.toHaveBeenCalled(); + expect(onFailureResult).not.toHaveBeenCalled(); + }); + + it("keeps health-pending retry semantics unchanged for existing callers (#8662)", () => { + const result = { + status: 1, + stdout: "", + stderr: "GATEWAY_HEALTH_TIMEOUT", + }; + const requestGatewaySupervisorActionImpl = vi.fn(() => result); + const sleepImpl = vi.fn(); + const onFailureResult = vi.fn(); + + expect( + waitForManagedGatewaySupervisor("restart-box", { + intervalSeconds: 2, + maxAttempts: 2, + onFailureResult, + requestGatewaySupervisorActionImpl, + sleepImpl, + }), + ).toBe(false); + + expect(requestGatewaySupervisorActionImpl).toHaveBeenCalledTimes(2); + expect(sleepImpl).toHaveBeenCalledOnce(); + expect(sleepImpl).toHaveBeenCalledWith(2); + expect(onFailureResult).toHaveBeenCalledWith(result); + }); + + it("waits through exact supervisor startup states before admitting health-pending (#8662)", () => { + const requestGatewaySupervisorActionImpl = vi + .fn() + .mockReturnValueOnce({ + status: 1, + stdout: "", + stderr: "SUPERVISOR_NOT_RUNNING\nNEMOCLAW_CONTROL_STAGE=discover-supervisor", + }) + .mockReturnValueOnce({ + status: 1, + stdout: "", + stderr: "PRIVILEGED_CONTROL_UNAVAILABLE", + }) + .mockReturnValueOnce({ status: 1, stdout: "", stderr: "GATEWAY_HEALTH_TIMEOUT" }); + const sleepImpl = vi.fn(); + + expect( + waitForManagedGatewaySupervisor("restart-box", { + acceptGatewayHealthPending: true, + intervalSeconds: 3, + maxAttempts: 3, + requestGatewaySupervisorActionImpl, + sleepImpl, + }), + ).toBe(true); + + expect(requestGatewaySupervisorActionImpl).toHaveBeenCalledTimes(3); + expect(sleepImpl).toHaveBeenCalledTimes(2); + expect(sleepImpl).toHaveBeenNthCalledWith(1, 3); + expect(sleepImpl).toHaveBeenNthCalledWith(2, 3); + }); + + it("rejects a missing-supervisor result from any other control stage (#8662)", () => { + const result = { + status: 1, + stdout: "", + stderr: "SUPERVISOR_NOT_RUNNING\nNEMOCLAW_CONTROL_STAGE=preflight", + }; + const requestGatewaySupervisorActionImpl = vi.fn(() => result); + const sleepImpl = vi.fn(); + const onFailureResult = vi.fn(); + + expect( + waitForManagedGatewaySupervisor("restart-box", { + acceptGatewayHealthPending: true, + onFailureResult, + requestGatewaySupervisorActionImpl, + sleepImpl, + }), + ).toBe(false); + + expect(requestGatewaySupervisorActionImpl).toHaveBeenCalledTimes(1); + expect(sleepImpl).not.toHaveBeenCalled(); + expect(onFailureResult).toHaveBeenCalledWith(result); + }); + + it("rejects decorated health output without retrying or hiding its result (#8662)", () => { + const result = { + status: 1, + stdout: "", + stderr: "GATEWAY_HEALTH_TIMEOUT\nunexpected controller diagnostic", + }; + const requestGatewaySupervisorActionImpl = vi.fn(() => result); + const sleepImpl = vi.fn(); + const onFailureResult = vi.fn(); + + expect( + waitForManagedGatewaySupervisor("restart-box", { + acceptGatewayHealthPending: true, + onFailureResult, + requestGatewaySupervisorActionImpl, + sleepImpl, + }), + ).toBe(false); + + expect(requestGatewaySupervisorActionImpl).toHaveBeenCalledTimes(1); + expect(sleepImpl).not.toHaveBeenCalled(); + expect(onFailureResult).toHaveBeenCalledWith(result); + }); +}); + describe("recreated sandbox OpenShell readiness", () => { afterEach(() => { vi.unstubAllEnvs(); diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 3adc1954473..4e50cbc55ce 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -140,8 +140,15 @@ export function executePrivilegedSandboxCommand( sandboxName: string, command: readonly string[], timeout: number, + expectedContainerId?: string, ): SandboxCommandResult | null { - const argv = privilegedSandboxExecArgv(sandboxName, [...command], false, true); + const argv = privilegedSandboxExecArgv( + sandboxName, + [...command], + false, + true, + expectedContainerId, + ); const result = dockerSpawnSync(argv, { cwd: ROOT, encoding: "utf-8", @@ -330,6 +337,20 @@ function isExactlyMissingManagedSupervisor(result: SandboxCommandResult | null): return lines.length === 1 && lines[0] === "SUPERVISOR_NOT_RUNNING"; } +function isExactlyStartingManagedSupervisor(result: SandboxCommandResult | null): boolean { + if (result === null || result.status !== 1 || result.stdout.trim() !== "") return false; + const lines = result.stderr + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + return ( + (lines.length === 1 && lines[0] === "SUPERVISOR_NOT_RUNNING") || + (lines.length === 2 && + lines[0] === "SUPERVISOR_NOT_RUNNING" && + lines[1] === "NEMOCLAW_CONTROL_STAGE=discover-supervisor") + ); +} + function isExactlyPendingManagedSupervisorControl(result: SandboxCommandResult | null): boolean { if (result === null || result.status !== 1 || result.stdout.trim() !== "") return false; const lines = result.stderr @@ -354,8 +375,10 @@ function isExactlyPendingManagedGatewayHealth(result: SandboxCommandResult | nul export function waitForManagedGatewaySupervisor( sandboxName: string, options: { + acceptGatewayHealthPending?: boolean; intervalSeconds?: number; maxAttempts?: number; + onFailureResult?: (result: SandboxCommandResult | null) => void; requestGatewaySupervisorActionImpl?: typeof executeGatewaySupervisorAction; sleepImpl?: (seconds: number) => void; } = {}, @@ -366,19 +389,29 @@ export function waitForManagedGatewaySupervisor( const intervalSeconds = options.intervalSeconds ?? 3; const maxAttempts = options.maxAttempts ?? 11; + let lastResult: SandboxCommandResult | null = null; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const result = requestGatewaySupervisorAction(sandboxName, "probe", OPENSHELL_PROBE_TIMEOUT_MS); + lastResult = result; if (hasGatewayRecoveryMarker(result)) return true; if ( - !isExactlyMissingManagedSupervisor(result) && + options.acceptGatewayHealthPending === true && + isExactlyPendingManagedGatewayHealth(result) + ) { + return true; + } + if ( + !isExactlyStartingManagedSupervisor(result) && !isExactlyRetryableManagedRecoveryFailure(result) && !isExactlyPendingManagedSupervisorControl(result) && !isExactlyPendingManagedGatewayHealth(result) ) { + options.onFailureResult?.(result); return false; } if (attempt < maxAttempts) sleep(intervalSeconds); } + options.onFailureResult?.(lastResult); return false; } @@ -432,6 +465,261 @@ type SandboxProcessRecovery = | { kind: "custom" } | { kind: "relaunched"; relaunch: ManagedSupervisorRelaunch }; +export type ManagedGatewayRecoveryFailure = ReturnType; + +function managedSupervisorProbeFailure( + sandboxName: string, + waitForSupervisor: typeof waitForManagedGatewaySupervisor, + requestGatewaySupervisorAction: typeof executeGatewaySupervisorAction, + acceptGatewayHealthPending: boolean, +): ManagedGatewayRecoveryFailure | null { + const failureResult: { current: SandboxCommandResult | null } = { current: null }; + const supervisorReady = waitForSupervisor(sandboxName, { + acceptGatewayHealthPending, + onFailureResult: (result) => { + failureResult.current = result; + }, + requestGatewaySupervisorActionImpl: requestGatewaySupervisorAction, + }); + return supervisorReady ? null : classifyGatewayRestartFailure(failureResult.current); +} + +/** + * Re-prove the authenticated managed gateway against the exact Docker + * container captured by lifecycle start. This is intentionally read-only and + * is used after connect-time inference/pairing reconciliation so `start` + * cannot return success after a same-name container swap. + */ +export function pinnedManagedGatewayProbeFailure( + sandboxName: string, + expectedContainerId: string, + { + requestPinnedGatewaySupervisorActionImpl = executeGatewaySupervisorActionPinned, + waitForManagedGatewaySupervisorImpl = waitForManagedGatewaySupervisor, + }: { + requestPinnedGatewaySupervisorActionImpl?: RequestPinnedGatewaySupervisorAction; + waitForManagedGatewaySupervisorImpl?: typeof waitForManagedGatewaySupervisor; + } = {}, +): ManagedGatewayRecoveryFailure | null { + try { + return managedSupervisorProbeFailure( + sandboxName, + waitForManagedGatewaySupervisorImpl, + (name, action, timeout = 210000) => + requestPinnedGatewaySupervisorActionImpl(name, action, timeout, expectedContainerId), + false, + ); + } catch { + return { + layer: "privileged control unavailable", + detail: "the pinned runtime identity changed during final managed gateway verification", + }; + } +} + +function strictManagedGatewayProbeFailureResult( + sandboxName: string, + { + required, + requestGatewaySupervisorAction, + waitForSupervisor, + wasRunning, + onRecoveryFailure, + onRecoveryFailureLayer, + }: { + required: boolean; + requestGatewaySupervisorAction: typeof executeGatewaySupervisorAction; + waitForSupervisor: typeof waitForManagedGatewaySupervisor; + wasRunning: boolean; + onRecoveryFailure?: (failure: ManagedGatewayRecoveryFailure) => void; + onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void; + }, +) { + if (!required) return null; + const failure = managedSupervisorProbeFailure( + sandboxName, + waitForSupervisor, + requestGatewaySupervisorAction, + false, + ); + if (!failure) return null; + onRecoveryFailure?.(failure); + onRecoveryFailureLayer?.(failure.layer); + return { + checked: true as const, + wasRunning, + recovered: false as const, + forwardRecovered: false as const, + forwardRecoveryFailed: undefined, + forwardRecoveryFailureDetail: undefined, + managedGatewayProbeFailed: true as const, + managedGatewayProbeFailureDetail: `${failure.layer}: ${failure.detail}`, + }; +} + +function createPreservedManagedForwardVerification( + sandboxName: string, + { + required, + requestGatewaySupervisorAction, + onRecoveryFailure, + onRecoveryFailureLayer, + }: { + required: boolean; + requestGatewaySupervisorAction: typeof executeGatewaySupervisorAction; + onRecoveryFailure?: (failure: ManagedGatewayRecoveryFailure) => void; + onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void; + }, +) { + const failureState: { current: ManagedGatewayRecoveryFailure | null } = { current: null }; + const guard = required + ? () => { + let probeResult: SandboxCommandResult | null = null; + try { + const confirmed = confirmRecoveredSandboxGatewayManaged(sandboxName, { + requestGatewaySupervisorActionImpl: (name, action) => { + probeResult = requestGatewaySupervisorAction( + name, + action, + OPENSHELL_PROBE_TIMEOUT_MS, + ); + return probeResult; + }, + }); + failureState.current = + confirmed === true ? null : classifyGatewayRestartFailure(probeResult); + return confirmed === true; + } catch { + failureState.current = { + layer: "privileged control unavailable", + detail: "the pinned container identity changed during host-forward verification", + }; + return false; + } + } + : undefined; + const failureResult = (wasRunning: boolean) => { + const failure = + failureState.current ?? + ({ + layer: "health timeout", + detail: "the pinned managed gateway health changed during host-forward verification", + } satisfies ManagedGatewayRecoveryFailure); + onRecoveryFailure?.(failure); + onRecoveryFailureLayer?.(failure.layer); + return { + checked: true as const, + wasRunning, + recovered: false as const, + forwardRecovered: false as const, + forwardRecoveryFailed: undefined, + forwardRecoveryFailureDetail: undefined, + managedGatewayProbeFailed: true as const, + managedGatewayProbeFailureDetail: `${failure.layer}: ${failure.detail}`, + }; + }; + return { + guard, + preferFailure(wasRunning: boolean, fallback: T) { + return failureState.current ? failureResult(wasRunning) : fallback; + }, + verifySuccess(wasRunning: boolean, success: T) { + return guard && !guard() ? failureResult(wasRunning) : success; + }, + }; +} + +function runningSandboxRecoveryPreflight( + sandboxName: string, + { + recoveryAgent, + requestGatewaySupervisorAction, + requireManagedHealth, + waitForSupervisor, + onRecoveryFailure, + onRecoveryFailureLayer, + }: { + recoveryAgent: ReturnType; + requestGatewaySupervisorAction: typeof executeGatewaySupervisorAction; + requireManagedHealth: boolean; + waitForSupervisor: typeof waitForManagedGatewaySupervisor; + onRecoveryFailure?: (failure: ManagedGatewayRecoveryFailure) => void; + onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void; + }, +) { + const enforcement = enforceHermesSecretBoundaryOnRunningGateway( + sandboxName, + recoveryAgent, + requestGatewaySupervisorAction, + ); + if (enforcement?.refused) { + return { + checked: true as const, + wasRunning: true, + recovered: false as const, + forwardRecovered: false as const, + forwardRecoveryFailed: undefined, + forwardRecoveryFailureDetail: undefined, + secretBoundaryRefused: true as const, + secretBoundaryReason: enforcement.reason, + }; + } + const mcpRefusal = processRecoveryMcpReconciliationRefusal(sandboxName, true); + if (mcpRefusal) return mcpRefusal; + return strictManagedGatewayProbeFailureResult(sandboxName, { + required: requireManagedHealth, + requestGatewaySupervisorAction, + waitForSupervisor, + wasRunning: true, + onRecoveryFailure, + onRecoveryFailureLayer, + }); +} + +function shouldUseManagedStartupRecovery( + sandboxName: string, + agent: ReturnType, + preserveContainer: boolean, +): boolean { + if (!preserveContainer) return false; + if (agent) return agent.name === "openclaw" || agent.name === "hermes"; + try { + const persistedAgent = registry.getSandbox(sandboxName)?.agent; + return !persistedAgent || persistedAgent === "openclaw" || persistedAgent === "hermes"; + } catch { + return false; + } +} + +function inspectManagedStartupRecovery( + sandboxName: string, + agent: ReturnType, + preserveContainer: boolean, + inspectGateway: typeof isSandboxGatewayRunning, + waitForSupervisor: typeof waitForManagedGatewaySupervisor, + requestGatewaySupervisorAction: typeof executeGatewaySupervisorAction, +): { + failure: ManagedGatewayRecoveryFailure | null; + managedStartupRecovery: boolean; + running: boolean | null; +} { + const running = inspectGateway(sandboxName); + const managedStartupRecovery = + running === null && shouldUseManagedStartupRecovery(sandboxName, agent, preserveContainer); + return { + failure: managedStartupRecovery + ? managedSupervisorProbeFailure( + sandboxName, + waitForSupervisor, + requestGatewaySupervisorAction, + true, + ) + : null, + managedStartupRecovery, + running, + }; +} + function recoverSandboxProcesses( sandboxName: string, { @@ -439,13 +727,15 @@ function recoverSandboxProcesses( requestGatewaySupervisorAction = executeGatewaySupervisorAction, requestPinnedGatewaySupervisorAction = executeGatewaySupervisorActionPinned, relaunchManagedSupervisorSessionImpl = relaunchManagedSupervisorSession, - onFailureLayer, + preserveContainer = false, + onFailure, }: { quiet?: boolean; requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction; requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction; relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession; - onFailureLayer?: (layer: GatewayRestartFailureLayer) => void; + preserveContainer?: boolean; + onFailure?: (failure: ManagedGatewayRecoveryFailure) => void; } = {}, ): SandboxProcessRecovery | null { const agent = agentRuntime.getSessionAgent(sandboxName); @@ -458,6 +748,7 @@ function recoverSandboxProcesses( error instanceof Error && error.message.trim() ? `Sandbox agent lookup failed: ${error.message}.` : "Sandbox agent lookup failed."; + onFailure?.({ layer: "unsupported agent", detail }); quiet || printGatewayRestartFailure(sandboxName, "unsupported agent", detail); return null; } @@ -483,8 +774,9 @@ function recoverSandboxProcesses( sleepSeconds(retryIntervalSeconds); } const failure = classifyGatewayRestartFailure(execResult); - onFailureLayer?.(failure.layer); + onFailure?.(failure); if ( + !preserveContainer && failure.layer === "supervisor not running" && isExactlyMissingManagedSupervisor(execResult) ) { @@ -522,6 +814,7 @@ function recoverSandboxProcesses( if (persistedAgent === "hermes") { if (!isHermesAgent(agent)) { const detail = "Hermes agent definition could not be loaded."; + onFailure?.({ layer: "unsupported agent", detail }); if (!quiet) printGatewayRestartFailure(sandboxName, "unsupported agent", detail); return null; } @@ -535,6 +828,7 @@ function recoverSandboxProcesses( // but fail closed for an explicit non-OpenClaw agent. if (persistedAgent && persistedAgent !== "openclaw" && !agent) { const detail = `${persistedAgent} agent definition could not be loaded.`; + onFailure?.({ layer: "unsupported agent", detail }); if (!quiet) printGatewayRestartFailure(sandboxName, "unsupported agent", detail); return null; } @@ -704,11 +998,11 @@ function recreatedSandboxOpenShellReadinessFailureDetail( const detail = (() => { switch (failure) { case "managed-health-definitive-failure": - return "the recreated sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; + return "the sandbox failed the managed health guard, so the primary dashboard/API host forward was not started"; case "managed-health-inconclusive-timeout": - return "the recreated sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; + return "the sandbox managed health guard stayed inconclusive within the readiness deadline, so the primary dashboard/API host forward was not started"; case "openshell-readiness-failure": - return "the recreated sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; + return "the sandbox did not become ready in OpenShell, so the primary dashboard/API host forward was not started"; } })(); const managedHealthResult = managedHealthFailureDetail @@ -847,6 +1141,59 @@ export function waitForRecreatedSandboxOpenShellReady( return waitForRecreatedSandboxOpenShellReadyResult(sandboxName, options).ready; } +type ManagedGatewayProbeRequest = ( + sandboxName: string, + action: "restart" | "recover" | "probe", + timeout?: number, +) => SandboxCommandResult | null; + +function managedStartupOpenShellReadinessFailureDetail( + sandboxName: string, + requestManagedProbe: ManagedGatewayProbeRequest, + waitForOpenShellReady: typeof waitForRecreatedSandboxOpenShellReady, +): string | null { + let managedHealthFailureDetail: string | null = null; + const readinessOptions: RecreatedSandboxOpenShellReadyOptions = { + beforeProbe: (timeoutMs) => { + let probeResult: SandboxCommandResult | null = null; + try { + const confirmed = confirmRecoveredSandboxGatewayManaged(sandboxName, { + requestGatewaySupervisorActionImpl: (name, action) => { + probeResult = requestManagedProbe(name, action, timeoutMs); + return probeResult; + }, + }); + if (confirmed === false) { + const failure = classifyGatewayRestartFailure(probeResult); + managedHealthFailureDetail = `${failure.layer}: ${failure.detail}`; + } else if (confirmed === true) { + managedHealthFailureDetail = null; + } + return confirmed; + } catch { + managedHealthFailureDetail = + "the existing sandbox identity changed during its managed readiness probe"; + return false; + } + }, + }; + const readiness = + waitForOpenShellReady === waitForRecreatedSandboxOpenShellReady + ? waitForRecreatedSandboxOpenShellReadyResult(sandboxName, readinessOptions) + : waitForOpenShellReady(sandboxName, readinessOptions) + ? ({ ready: true } as const) + : ({ failure: "openshell-readiness-failure", ready: false } as const); + return readiness.ready + ? null + : recreatedSandboxOpenShellReadinessFailureDetail( + readiness.failure, + "openshellError" in readiness ? readiness.openshellError : undefined, + readiness.failure === "managed-health-definitive-failure" + ? (managedHealthFailureDetail ?? undefined) + : undefined, + ); +} + function gatewayRecoveryTimeoutSeconds( agent: ReturnType, ): number { @@ -1048,6 +1395,10 @@ function isHermesAgent( return !!agent && agent.name === "hermes"; } +function isTerminalRecoveryAgent(agent: ReturnType): boolean { + return !!agent && !agentRuntime.hasGatewayRuntime(agent); +} + /** * Detect and recover from a sandbox that survived a gateway restart but * whose OpenClaw processes are not running. Also re-establishes the @@ -1057,33 +1408,51 @@ function isHermesAgent( * `onRecoveryFailureLayer` reports the classified managed-restart failure so a * quiet caller (`recover`, `connect --probe-only`) can still explain why * recovery is not retryable instead of printing a generic "check the gateway - * log". The result shape is unchanged so existing callers keep their contract. + * log". `onRecoveryFailure` retains the sanitized controller detail for + * startup callers without changing the long-standing result shape. */ +export interface SandboxProcessRecoveryOptions { + /** Immutable direct-container identity captured by lifecycle `start`. */ + expectedContainerId?: string; + quiet?: boolean; + requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction; + requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction; + relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession; + isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning; + waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady; + waitForManagedGatewaySupervisorImpl?: typeof waitForManagedGatewaySupervisor; + isWsl?: boolean; + onRecoveryFailure?: (failure: ManagedGatewayRecoveryFailure) => void; + onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void; + /** + * Recover only through the supervisor in the already-started container. + * This mode never authorizes the legacy transactional container-relaunch + * fallback and may use the pinned managed controller while OpenShell exec is + * still gated on Provisioning. + */ + preserveContainer?: boolean; +} + function checkAndRecoverSandboxProcessesWithoutHostLock( sandboxName: string, { + expectedContainerId, quiet = false, requestGatewaySupervisorAction = executeGatewaySupervisorAction, requestPinnedGatewaySupervisorAction = executeGatewaySupervisorActionPinned, relaunchManagedSupervisorSessionImpl = relaunchManagedSupervisorSession, isSandboxGatewayRunningImpl = isSandboxGatewayRunning, waitForRecreatedSandboxOpenShellReadyImpl = waitForRecreatedSandboxOpenShellReady, + waitForManagedGatewaySupervisorImpl = waitForManagedGatewaySupervisor, isWsl: isWslOverride, + onRecoveryFailure, onRecoveryFailureLayer, - }: { - quiet?: boolean; - requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction; - requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction; - relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession; - isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning; - waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady; - isWsl?: boolean; - onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void; - } = {}, + preserveContainer = false, + }: SandboxProcessRecoveryOptions = {}, ) { const recoveryAgent = agentRuntime.getSessionAgent(sandboxName); const recoveryDisplayName = recoveryAgentDisplayName(sandboxName, recoveryAgent); - if (recoveryAgent && !agentRuntime.hasGatewayRuntime(recoveryAgent)) { + if (isTerminalRecoveryAgent(recoveryAgent)) { return { checked: true, wasRunning: null, @@ -1092,30 +1461,69 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( runtime: "terminal" as const, }; } - const running = isSandboxGatewayRunningImpl(sandboxName); - if (running === null) { + const requestManagedAction: typeof executeGatewaySupervisorAction = expectedContainerId + ? (name, action, timeout = 210000) => + requestPinnedGatewaySupervisorAction(name, action, timeout, expectedContainerId) + : requestGatewaySupervisorAction; + const requirePreservedManagedHealth = shouldUseManagedStartupRecovery( + sandboxName, + recoveryAgent, + preserveContainer, + ); + const preservedManagedForwardVerification = createPreservedManagedForwardVerification( + sandboxName, + { + required: requirePreservedManagedHealth, + requestGatewaySupervisorAction: requestManagedAction, + onRecoveryFailure, + onRecoveryFailureLayer, + }, + ); + const preservedManagedForwardGuard = preservedManagedForwardVerification.guard; + const startupInspection = inspectManagedStartupRecovery( + sandboxName, + recoveryAgent, + preserveContainer, + isSandboxGatewayRunningImpl, + waitForManagedGatewaySupervisorImpl, + requestManagedAction, + ); + const { managedStartupRecovery, running } = startupInspection; + if (startupInspection.failure) { + // OpenShell v0.0.99+ keeps exec gated while its authenticated supervisor + // session is still Provisioning. Use the registry-pinned root controller + // as the startup barrier so recovery does not wait on the readiness state + // it is responsible for establishing. A bare managed health timeout is + // accepted only as proof of the pinned OpenShell PID 1, its non-root + // NemoClaw supervisor child, and successful managed preflight. It does not + // prove OpenShell session readiness; the read-only readiness gate after + // gateway recovery does that before any host-forward mutation. + onRecoveryFailure?.(startupInspection.failure); + onRecoveryFailureLayer?.(startupInspection.failure.layer); + return { + checked: false, + wasRunning: null, + recovered: false, + forwardRecovered: false, + managedSupervisorUnavailable: true, + managedSupervisorFailureDetail: `${startupInspection.failure.layer}: ${startupInspection.failure.detail}`, + }; + } + if (running === null && !managedStartupRecovery) { return { checked: false, wasRunning: null, recovered: false, forwardRecovered: false }; } const recoveryPort = resolveSandboxDashboardPort(sandboxName); - if (running) { - const enforcement = enforceHermesSecretBoundaryOnRunningGateway( - sandboxName, - recoveryAgent, - requestGatewaySupervisorAction, - ); - if (enforcement?.refused) { - return { - checked: true, - wasRunning: true, - recovered: false, - forwardRecovered: false, - secretBoundaryRefused: true, - secretBoundaryReason: enforcement.reason, - }; - } - const mcpRefusal = processRecoveryMcpReconciliationRefusal(sandboxName, true); - if (mcpRefusal) return mcpRefusal; - } + const runningPreflightFailure = running + ? runningSandboxRecoveryPreflight(sandboxName, { + recoveryAgent, + requestGatewaySupervisorAction: requestManagedAction, + requireManagedHealth: requirePreservedManagedHealth, + waitForSupervisor: waitForManagedGatewaySupervisorImpl, + onRecoveryFailure, + onRecoveryFailureLayer, + }) + : null; + if (runningPreflightFailure) return runningPreflightFailure; if (running) { // Gateway is alive but the host-side forward can still be dead or // owned by another sandbox. Probe and re-establish only when @@ -1127,13 +1535,26 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( console.log(` Dashboard port forward to '${sandboxName}' is missing or dead.`); console.log(" Re-establishing..."); } - const forwardRecovered = ensureSandboxPortForward(sandboxName, { isWsl: isWslOverride }); - const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName); - const messagingForwardRecovered = recoverMessagingHostForward(sandboxName, { quiet }); + const forwardRecovered = ensureSandboxPortForward(sandboxName, { + afterSuccess: preservedManagedForwardGuard, + beforeStart: preservedManagedForwardGuard, + isWsl: isWslOverride, + }); + const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName, { + afterSuccess: preservedManagedForwardGuard, + beforeStart: preservedManagedForwardGuard, + }); + const messagingForwardRecovered = recoverMessagingHostForward(sandboxName, { + afterSuccess: preservedManagedForwardGuard, + beforeStart: preservedManagedForwardGuard, + quiet, + }); const declaredForwardsRecovered = recoverDeclaredAgentForwardPorts( sandboxName, recoveryPort, { + afterSuccess: preservedManagedForwardGuard, + beforeStart: preservedManagedForwardGuard, quiet, }, ); @@ -1157,33 +1578,33 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } } if (!forwardRecovered) { - return { - checked: true, + return preservedManagedForwardVerification.preferFailure(true, { + checked: true as const, wasRunning: true, - recovered: false, - forwardRecovered: false, - forwardRecoveryFailed: true, + recovered: false as const, + forwardRecovered: false as const, + forwardRecoveryFailed: true as const, forwardRecoveryFailureDetail: "the primary dashboard/API host forward could not be re-established", - }; + }); } if (auxiliaryFailureDetail !== null) { if (!quiet) console.error(` ${auxiliaryFailureDetail}.`); - return { - checked: true, + return preservedManagedForwardVerification.preferFailure(true, { + checked: true as const, wasRunning: true, - recovered: false, - forwardRecovered: false, - forwardRecoveryFailed: true, + recovered: false as const, + forwardRecovered: false as const, + forwardRecoveryFailed: true as const, forwardRecoveryFailureDetail: auxiliaryFailureDetail, - }; + }); } - return { - checked: true, + return preservedManagedForwardVerification.verifySuccess(true, { + checked: true as const, wasRunning: true, - recovered: false, + recovered: false as const, forwardRecovered: forwardRecovered || anyAuxiliaryRecovered(auxiliaryResults), - }; + }); } if (forwardHealthy === "occupied") { if (!quiet) { @@ -1212,9 +1633,18 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( "the primary dashboard/API host forward could not be verified because OpenShell forward state was unavailable", }; } - const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName); - const messagingForwardRecovered = recoverMessagingHostForward(sandboxName, { quiet }); + const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName, { + afterSuccess: preservedManagedForwardGuard, + beforeStart: preservedManagedForwardGuard, + }); + const messagingForwardRecovered = recoverMessagingHostForward(sandboxName, { + afterSuccess: preservedManagedForwardGuard, + beforeStart: preservedManagedForwardGuard, + quiet, + }); const declaredForwardsRecovered = recoverDeclaredAgentForwardPorts(sandboxName, recoveryPort, { + afterSuccess: preservedManagedForwardGuard, + beforeStart: preservedManagedForwardGuard, quiet, }); const auxiliaryResults = [ @@ -1225,21 +1655,21 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( const auxiliaryFailureDetail = auxiliaryRecoveryFailureDetail(auxiliaryResults); if (auxiliaryFailureDetail !== null) { if (!quiet) console.error(` ${auxiliaryFailureDetail}.`); - return { - checked: true, + return preservedManagedForwardVerification.preferFailure(true, { + checked: true as const, wasRunning: true, - recovered: false, - forwardRecovered: false, - forwardRecoveryFailed: true, + recovered: false as const, + forwardRecovered: false as const, + forwardRecoveryFailed: true as const, forwardRecoveryFailureDetail: auxiliaryFailureDetail, - }; + }); } - return { - checked: true, + return preservedManagedForwardVerification.verifySuccess(true, { + checked: true as const, wasRunning: true, - recovered: false, + recovered: false as const, forwardRecovered: anyAuxiliaryRecovered(auxiliaryResults), - }; + }); } // Gateway not running — attempt recovery @@ -1251,16 +1681,21 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( console.log(" Recovering..."); } - let managedRecoveryFailureLayer: GatewayRestartFailureLayer | null = null; + const managedRecoveryFailureState: { current: ManagedGatewayRecoveryFailure | null } = { + current: null, + }; const recovery = recoverSandboxProcesses(sandboxName, { quiet, - requestGatewaySupervisorAction, + requestGatewaySupervisorAction: requestManagedAction, requestPinnedGatewaySupervisorAction, relaunchManagedSupervisorSessionImpl, - onFailureLayer: (layer) => { - managedRecoveryFailureLayer = layer; + preserveContainer, + onFailure: (failure) => { + managedRecoveryFailureState.current = failure; }, }); + const managedRecoveryFailure = managedRecoveryFailureState.current; + const managedRecoveryFailureLayer = managedRecoveryFailure?.layer ?? null; if (recovery !== null) { const withManagedControlCompletion = ( result: T, @@ -1272,7 +1707,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( const requestManagedProbe = relaunch ? (name: string, action: "restart" | "recover" | "probe", timeout = 210000) => requestPinnedGatewaySupervisorAction(name, action, timeout, relaunch.containerId) - : requestGatewaySupervisorAction; + : requestManagedAction; let relaunchedIdentityRejected = false; let relaunchedManagedHealthFailureDetail: string | null = null; const confirmRelaunchedManagedHealth = relaunch @@ -1354,6 +1789,12 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( ); } onRecoveryFailureLayer?.(managedRecoveryFailureLayer); + onRecoveryFailure?.( + managedRecoveryFailure ?? { + layer: "health timeout", + detail: "the managed gateway did not pass health verification", + }, + ); if (relaunchedManagedHealthFailureDetail) { return { checked: true, @@ -1366,6 +1807,23 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } + if (managedStartupRecovery) { + const readinessFailureDetail = managedStartupOpenShellReadinessFailureDetail( + sandboxName, + requestManagedProbe, + waitForRecreatedSandboxOpenShellReadyImpl, + ); + if (readinessFailureDetail) { + return { + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + openshellReadinessFailed: true, + openshellReadinessFailureDetail: readinessFailureDetail, + }; + } + } // State restore crosses the OpenShell SSH boundary. Prove the replacement // is both identity-pinned and registered before asking finalize(true) to // mutate it; otherwise a slow control-plane handoff can make a healthy @@ -1456,9 +1914,20 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } const mcpRefusal = processRecoveryMcpReconciliationRefusal(sandboxName, false); if (mcpRefusal) return mcpRefusal; + const managedProbeFailure = strictManagedGatewayProbeFailureResult(sandboxName, { + required: requirePreservedManagedHealth, + requestGatewaySupervisorAction: requestManagedProbe, + waitForSupervisor: waitForManagedGatewaySupervisorImpl, + wasRunning: false, + onRecoveryFailure, + onRecoveryFailureLayer, + }); + if (managedProbeFailure) return managedProbeFailure; + const managedForwardGuard = + confirmRelaunchedManagedHealthForForward ?? preservedManagedForwardGuard; const forwardRecovered = ensureSandboxPortForward(sandboxName, { - afterSuccess: confirmRelaunchedManagedHealthForForward ?? undefined, - beforeStart: confirmRelaunchedManagedHealthForForward ?? undefined, + afterSuccess: managedForwardGuard, + beforeStart: managedForwardGuard, isWsl: isWslOverride, }); if (!forwardRecovered && relaunchedIdentityRejected) { @@ -1472,9 +1941,18 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( "the primary dashboard/API host forward could not be re-established", }); } - const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName); - const messagingForwardRecovered = recoverMessagingHostForward(sandboxName, { quiet }); + const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName, { + afterSuccess: managedForwardGuard, + beforeStart: managedForwardGuard, + }); + const messagingForwardRecovered = recoverMessagingHostForward(sandboxName, { + afterSuccess: managedForwardGuard, + beforeStart: managedForwardGuard, + quiet, + }); const declaredForwardsRecovered = recoverDeclaredAgentForwardPorts(sandboxName, recoveryPort, { + afterSuccess: managedForwardGuard, + beforeStart: managedForwardGuard, quiet, }); const auxiliaryResults = [ @@ -1495,33 +1973,42 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } } if (!forwardRecovered) { - return withManagedControlCompletion({ - checked: true, - wasRunning: false, - recovered: true, - forwardRecovered: false, - forwardRecoveryFailed: true, - forwardRecoveryFailureDetail: - "the primary dashboard/API host forward could not be re-established", - }); + return preservedManagedForwardVerification.preferFailure( + false, + withManagedControlCompletion({ + checked: true as const, + wasRunning: false, + recovered: true as const, + forwardRecovered: false as const, + forwardRecoveryFailed: true as const, + forwardRecoveryFailureDetail: + "the primary dashboard/API host forward could not be re-established", + }), + ); } if (auxiliaryFailureDetail !== null) { if (!quiet) console.error(` ${auxiliaryFailureDetail}.`); - return withManagedControlCompletion({ - checked: true, - wasRunning: false, - recovered: true, - forwardRecovered: false, - forwardRecoveryFailed: true, - forwardRecoveryFailureDetail: auxiliaryFailureDetail, - }); + return preservedManagedForwardVerification.preferFailure( + false, + withManagedControlCompletion({ + checked: true as const, + wasRunning: false, + recovered: true as const, + forwardRecovered: false as const, + forwardRecoveryFailed: true as const, + forwardRecoveryFailureDetail: auxiliaryFailureDetail, + }), + ); } - return withManagedControlCompletion({ - checked: true, - wasRunning: false, - recovered: true, - forwardRecovered: forwardRecovered || anyAuxiliaryRecovered(auxiliaryResults), - }); + return preservedManagedForwardVerification.verifySuccess( + false, + withManagedControlCompletion({ + checked: true as const, + wasRunning: false, + recovered: true as const, + forwardRecovered: forwardRecovered || anyAuxiliaryRecovered(auxiliaryResults), + }), + ); } if (!quiet) { console.error(` Could not restart ${recoveryDisplayName} gateway automatically.`); @@ -1529,21 +2016,18 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } onRecoveryFailureLayer?.(managedRecoveryFailureLayer); + onRecoveryFailure?.( + managedRecoveryFailure ?? { + layer: "launch failure", + detail: "the managed gateway recovery did not return a completion result", + }, + ); return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } export function checkAndRecoverSandboxProcesses( sandboxName: string, - options: { - quiet?: boolean; - requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction; - requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction; - relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession; - isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning; - waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady; - isWsl?: boolean; - onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void; - } = {}, + options: SandboxProcessRecoveryOptions = {}, ) { return withTimerBoundShieldsMutationLock(sandboxName, "gateway process recovery", () => checkAndRecoverSandboxProcessesWithoutHostLock(sandboxName, options), diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index 08264353c3c..0804122f946 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -1,35 +1,54 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as agentRuntime from "../../agent/runtime"; import { createDockerRuntimeProviderBundle, createKubernetesRuntimeProviderBundle, type DockerRuntimeProviderDependencies, } from "../../onboard/runtime-provider/docker"; import { createRuntimeProviderBundleRegistry } from "../../onboard/runtime-provider/registry"; +import * as shields from "../../shields"; import type { SandboxEntry } from "../../state/registry"; +import * as persistedRegistry from "../../state/registry"; +import * as connect from "./connect"; +import * as processRecovery from "./process-recovery"; import { - assertSandboxStartupRecovery, restoreStoppedSandboxStartupState, type SandboxStartDeps, + sandboxStartDependencies, startSandbox, } from "./start"; -const RUNNING_RECOVERY = { - checked: true, - wasRunning: true, - recovered: false, - forwardRecovered: false, -} as const; -const SECRET_BOUNDARY_PRIVATE_DETAIL = "raw-secret" as const; -const PRIVATE_RECOVERY_DETAIL = "raw diagnostic remains private"; +const ORIGINAL_CONTAINER_ID = "a".repeat(64); + +type StartupRecoveryResult = ReturnType; function sandbox(values: Partial = {}): SandboxEntry { return { name: "my-sandbox", ...values }; } +function runningStartupState(): ReturnType { + return { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + }; +} + +function stubProductionStartupRecovery(result: StartupRecoveryResult) { + const check = vi + .spyOn(processRecovery, "checkAndRecoverSandboxProcesses") + .mockReturnValue(result); + const restoreAccess = vi + .spyOn(shields, "restoreLockedStateDirStartupAccess") + .mockImplementation(() => undefined); + return { check, restoreAccess }; +} + function harness(overrides: Partial = {}) { const getSandbox = vi.fn>(() => sandbox()); const isDockerRuntimeDown = vi.fn( @@ -41,6 +60,7 @@ function harness(overrides: Partial = {}) { DockerRuntimeProviderDependencies["findLabeledSandboxContainers"] >(() => [ { + containerId: ORIGINAL_CONTAINER_ID, name: "openshell-my-sandbox", status: "Exited (0) 2 hours ago", running: false, @@ -50,6 +70,7 @@ function harness(overrides: Partial = {}) { () => ({ recovered: true, via: "started-stopped-original", + containerId: ORIGINAL_CONTAINER_ID, containerName: "openshell-my-sandbox", }), ); @@ -59,8 +80,8 @@ function harness(overrides: Partial = {}) { const verifyGateway = vi.fn>(() => Promise.resolve(), ); - const restoreStartupState = vi.fn>( - () => RUNNING_RECOVERY, + const restoreStartupState = vi.fn>(() => + runningStartupState(), ); const log = vi.fn<(message: string) => void>(); const runtimeProviders = createRuntimeProviderBundleRegistry([ @@ -99,9 +120,20 @@ function harness(overrides: Partial = {}) { } describe("startSandbox", () => { + beforeEach(() => { + vi.spyOn(sandboxStartDependencies, "loadConnect").mockReturnValue(connect); + vi.spyOn(sandboxStartDependencies, "loadShields").mockReturnValue(shields); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }); + it("restores sealed access before recovering sandbox processes (#8112)", () => { const restoreAccess = vi.fn(); - const restoreProcesses = vi.fn(() => RUNNING_RECOVERY); + const recovery = runningStartupState(); + const restoreProcesses = vi.fn(() => recovery); const result = restoreStoppedSandboxStartupState("my-sandbox", { agent: "openclaw", @@ -109,17 +141,18 @@ describe("startSandbox", () => { restoreProcessState: restoreProcesses, }); - expect(result).toBe(RUNNING_RECOVERY); - expect(restoreAccess).toHaveBeenCalledWith("my-sandbox"); - expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox", true); + expect(restoreAccess).toHaveBeenCalledWith("my-sandbox", undefined); + expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox"); expect(restoreAccess.mock.invocationCallOrder[0]).toBeLessThan( restoreProcesses.mock.invocationCallOrder[0], ); + expect(result).toBe(recovery); }); it("keeps Hermes sealed state untouched while recovering sandbox processes (#8112)", () => { const restoreAccess = vi.fn(); - const restoreProcesses = vi.fn(() => RUNNING_RECOVERY); + const recovery = runningStartupState(); + const restoreProcesses = vi.fn(() => recovery); const result = restoreStoppedSandboxStartupState("my-sandbox", { agent: "hermes", @@ -127,20 +160,9 @@ describe("startSandbox", () => { restoreProcessState: restoreProcesses, }); - expect(result).toBe(RUNNING_RECOVERY); expect(restoreAccess).not.toHaveBeenCalled(); - expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox", true); - }); - - it("does not force managed recovery for custom agents (#8662)", () => { - const restoreProcesses = vi.fn(() => RUNNING_RECOVERY); - - restoreStoppedSandboxStartupState("my-sandbox", { - agent: "custom-agent", - restoreProcessState: restoreProcesses, - }); - - expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox", false); + expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox"); + expect(result).toBe(recovery); }); it("restores startup state before probing readiness after a stopped container starts (#8112)", async () => { @@ -152,8 +174,15 @@ describe("startSandbox", () => { expect(h.recoverDockerDriverSandbox).toHaveBeenCalledWith("my-sandbox", { readiness: "runtime-running", }); - expect(h.restoreStartupState).toHaveBeenCalledWith("my-sandbox"); - expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); + expect(h.restoreStartupState).toHaveBeenCalledWith( + "my-sandbox", + expect.objectContaining({ expectedContainerId: ORIGINAL_CONTAINER_ID }), + ); + expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox", { + expectedContainerId: ORIGINAL_CONTAINER_ID, + preserveContainer: true, + requirePinnedManagedGatewayProof: true, + }); expect(h.recoverDockerDriverSandbox.mock.invocationCallOrder[0]).toBeLessThan( h.restoreStartupState.mock.invocationCallOrder[0], ); @@ -162,6 +191,47 @@ describe("startSandbox", () => { ); }); + it("keeps the original container pinned through the production final probe (#8662)", async () => { + const connectSandbox = vi.spyOn(connect, "connectSandbox").mockResolvedValue(undefined); + const finalManagedProbe = vi + .spyOn(connect, "pinnedManagedGatewayProbeFailure") + .mockReturnValue(null); + const h = harness({ verifyGateway: undefined }); + + await expect(startSandbox("my-sandbox", h.deps)).resolves.toEqual({ exitCode: 0 }); + + expect(connectSandbox).toHaveBeenCalledExactlyOnceWith("my-sandbox", { + expectedContainerId: ORIGINAL_CONTAINER_ID, + preserveContainer: true, + probeOnly: true, + }); + expect(finalManagedProbe).toHaveBeenCalledExactlyOnceWith("my-sandbox", ORIGINAL_CONTAINER_ID); + expect(connectSandbox.mock.invocationCallOrder[0]).toBeLessThan( + finalManagedProbe.mock.invocationCallOrder[0], + ); + }, 15_000); + + it("fails after connect-time reconciliation if the original container identity changes (#8662)", async () => { + const connectSandbox = vi.spyOn(connect, "connectSandbox").mockResolvedValue(undefined); + const finalManagedProbe = vi + .spyOn(connect, "pinnedManagedGatewayProbeFailure") + .mockReturnValue({ + layer: "privileged control unavailable", + detail: "container identity changed; Authorization: Bearer opaque-post-connect-token", + }); + const h = harness({ verifyGateway: undefined }); + + await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow( + /final managed gateway health.*container identity changed.*/iu, + ); + + expect(connectSandbox).toHaveBeenCalledOnce(); + expect(finalManagedProbe).toHaveBeenCalledExactlyOnceWith("my-sandbox", ORIGINAL_CONTAINER_ID); + expect(connectSandbox.mock.invocationCallOrder[0]).toBeLessThan( + finalManagedProbe.mock.invocationCallOrder[0], + ); + }, 15_000); + it("attempts startup restoration again when start is rerun after a failure (#8112)", async () => { const h = harness(); h.restoreStartupState.mockImplementationOnce(() => { @@ -181,130 +251,277 @@ describe("startSandbox", () => { ); }); - it("fails before readiness when the managed startup result is inconclusive (#8662)", async () => { + it("directs expired Shields startup recovery to the command that can complete it (#8662)", async () => { const h = harness(); - h.restoreStartupState.mockReturnValue({ - checked: false, - wasRunning: null, - recovered: false, - forwardRecovered: false, + h.restoreStartupState.mockImplementation(() => { + throw Object.assign(new Error("expired auto-restore"), { + code: "NEMOCLAW_SHIELDS_AUTO_RESTORE_REQUIRED", + }); }); await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow( - "managed gateway could not be inspected through the trusted container boundary", + /nemoclaw my-sandbox shields up.*retry `nemoclaw my-sandbox start`/iu, ); + expect(h.verifyGateway).not.toHaveBeenCalled(); }); - it("keeps custom-agent readiness authoritative when recovery is inconclusive (#8662)", async () => { - const h = harness(); - h.getSandbox.mockReturnValue(sandbox({ agent: "custom-agent" })); - h.restoreStartupState.mockReturnValue({ - checked: false, - wasRunning: null, + it("propagates the actual managed recovery failure before readiness verification (#8662)", async () => { + const production = stubProductionStartupRecovery({ + checked: true, + wasRunning: false, recovered: false, forwardRecovered: false, }); + const h = harness({ restoreStartupState: undefined }); + h.getSandbox.mockReturnValue(sandbox({ agent: "hermes" })); - await expect(startSandbox("my-sandbox", h.deps)).resolves.toEqual({ exitCode: 0 }); - expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); + await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow( + /startup recovery failed.*nemoclaw my-sandbox recover/iu, + ); + + expect(production.check).toHaveBeenCalledWith( + "my-sandbox", + expect.objectContaining({ preserveContainer: true, quiet: true }), + ); + expect(production.restoreAccess).not.toHaveBeenCalled(); + expect(h.verifyGateway).not.toHaveBeenCalled(); }); - it("verifies readiness after managed gateway recovery succeeds (#8662)", async () => { - const h = harness(); - h.restoreStartupState.mockReturnValue({ + it.each([ + "openclaw", + "hermes", + ] as const)("threads %s managed recovery success through production restoration before final verification (#8662)", async (agent) => { + const managedResult = { checked: true, wasRunning: false, recovered: true, forwardRecovered: true, - }); + managedControlCompletion: { + disposition: "ok", + oldPid: 0, + newPid: 4242, + }, + } as StartupRecoveryResult; + const production = stubProductionStartupRecovery(managedResult); + const h = harness({ restoreStartupState: undefined }); + h.getSandbox.mockReturnValue(sandbox({ agent })); await expect(startSandbox("my-sandbox", h.deps)).resolves.toEqual({ exitCode: 0 }); - expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); - expect(h.restoreStartupState.mock.invocationCallOrder[0]).toBeLessThan( + + expect(h.restoreStartupState).not.toHaveBeenCalled(); + expect(production.check).toHaveBeenCalledWith( + "my-sandbox", + expect.objectContaining({ preserveContainer: true, quiet: true }), + ); + expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox", { + expectedContainerId: ORIGINAL_CONTAINER_ID, + preserveContainer: true, + requirePinnedManagedGatewayProof: true, + }); + expect(h.recoverDockerDriverSandbox.mock.invocationCallOrder[0]).toBeLessThan( + production.check.mock.invocationCallOrder[0], + ); + expect(production.check.mock.invocationCallOrder[0]).toBeLessThan( h.verifyGateway.mock.invocationCallOrder[0], ); + expect(production.restoreAccess.mock.calls).toEqual( + agent === "openclaw" ? [["my-sandbox", ORIGINAL_CONTAINER_ID]] : [], + ); + expect( + agent === "hermes" || + production.restoreAccess.mock.invocationCallOrder[0] < + production.check.mock.invocationCallOrder[0], + ).toBe(true); }); - it.each([ - [ - "secret-boundary refusal", + it("surfaces real preserved-controller failures for both managed agents through public start (#8662)", async () => { + const getSessionAgent = vi.spyOn(agentRuntime, "getSessionAgent"); + const getPersistedSandbox = vi.spyOn(persistedRegistry, "getSandbox"); + + for (const agent of ["openclaw", "hermes"] as const) { + const sandboxName = agent === "openclaw" ? "oc-restart" : "hm-restart"; + const entry = sandbox({ name: sandboxName, agent, openshellDriver: "docker" }); + getSessionAgent.mockReturnValue({ + name: agent, + displayName: agent === "hermes" ? "Hermes Agent" : "OpenClaw", + forwardPort: agent === "hermes" ? 8642 : 18789, + healthProbe: { + port: agent === "hermes" ? 8642 : 18789, + timeout_seconds: 30, + url: `http://127.0.0.1:${agent === "hermes" ? 8642 : 18789}/health`, + }, + runtime: { kind: "gateway" }, + } as never); + getPersistedSandbox.mockReturnValue(entry); + const controller = vi.fn( + ( + _name: string, + action: "restart" | "recover" | "probe", + _timeout: number, + expectedContainerId?: string, + ) => { + expect(expectedContainerId).toBe(ORIGINAL_CONTAINER_ID); + return action === "recover" + ? { + status: 1, + stdout: "", + stderr: + "SUPERVISOR_UNAVAILABLE\nNEMOCLAW_CONTROL_STAGE=preflight\nAuthorization: Bearer opaque-token-8662", + } + : { status: 1, stdout: "", stderr: "GATEWAY_HEALTH_TIMEOUT" }; + }, + ); + const relaunch = vi.fn(() => { + throw new Error("preserved start attempted to replace its container"); + }); + const restoreStartupState = vi.fn>( + (name, options = {}) => + connect.restoreSandboxStartupState(name, { + ...options, + isSandboxGatewayRunningImpl: () => null, + requestPinnedGatewaySupervisorAction: controller, + relaunchManagedSupervisorSessionImpl: relaunch, + waitForManagedGatewaySupervisorImpl: () => true, + }), + ); + const h = harness({ restoreStartupState }); + h.getSandbox.mockReturnValue(entry); + + const failure = await startSandbox(sandboxName, h.deps).catch((error: unknown) => error); + + expect(failure).toBeInstanceOf(Error); + const failureMessage = (failure as Error).message; + expect(failureMessage).toMatch( + /managed gateway recovery: supervisor unavailable.*NEMOCLAW_CONTROL_STAGE=preflight/iu, + ); + expect(failureMessage).toContain(""); + expect(failureMessage).not.toContain("opaque-token-8662"); + expect(restoreStartupState).toHaveBeenCalledWith( + sandboxName, + expect.objectContaining({ expectedContainerId: ORIGINAL_CONTAINER_ID }), + ); + expect(controller).toHaveBeenCalledExactlyOnceWith( + sandboxName, + "recover", + 210000, + ORIGINAL_CONTAINER_ID, + ); + expect(relaunch).not.toHaveBeenCalled(); + expect(h.verifyGateway).not.toHaveBeenCalled(); + } + }); + + it("keeps a missing custom-agent manifest outside managed OpenClaw recovery (#8662)", async () => { + const entry = sandbox({ agent: "custom-agent", openshellDriver: "docker" }); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null); + vi.spyOn(persistedRegistry, "getSandbox").mockReturnValue(entry); + const controller = vi.fn(() => { + throw new Error("custom agent crossed the managed controller boundary"); + }); + const h = harness({ + startupRecovery: { + isSandboxGatewayRunningImpl: () => null, + requestGatewaySupervisorAction: controller, + }, + }); + h.getSandbox.mockReturnValue(entry); + + await expect(startSandbox("my-sandbox", h.deps)).resolves.toEqual({ exitCode: 0 }); + + expect(controller).not.toHaveBeenCalled(); + expect(h.restoreStartupState).toHaveBeenCalledWith("my-sandbox"); + expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); + }); + + it.each( + (["openclaw", "hermes"] as const).flatMap((agent) => [ { - checked: true, - wasRunning: false, - recovered: false, - forwardRecovered: false, - secretBoundaryRefused: true, - secretBoundaryReason: SECRET_BOUNDARY_PRIVATE_DETAIL, + agent, + label: "inspection failure", + result: { + checked: false, + wasRunning: null, + recovered: false, + forwardRecovered: false, + } as StartupRecoveryResult, + expected: /inspection.*could not be inspected/iu, + privateDetail: undefined, }, - "secret-boundary check refused", - SECRET_BOUNDARY_PRIVATE_DETAIL, - ], - [ - "MCP reconciliation refusal", { - checked: true, - wasRunning: false, - recovered: false, - forwardRecovered: false, - mcpReconciliationRefused: true, - mcpReconciliationReason: PRIVATE_RECOVERY_DETAIL, + agent, + label: "secret-boundary refusal", + result: { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, + secretBoundaryRefused: true, + secretBoundaryReason: "raw-secret", + } as StartupRecoveryResult, + expected: /secret boundary.*agent secret boundary refused recovery/iu, + privateDetail: "raw-secret", }, - "MCP configuration does not match", - PRIVATE_RECOVERY_DETAIL, - ], - [ - "forward recovery failure", { - checked: true, - wasRunning: false, - recovered: true, - forwardRecovered: false, - forwardRecoveryFailed: true, - forwardRecoveryFailureDetail: PRIVATE_RECOVERY_DETAIL, + agent, + label: "MCP reconciliation refusal", + result: { + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + mcpReconciliationRefused: true, + mcpReconciliationReason: "registered MCP intent no longer matches", + } as StartupRecoveryResult, + expected: /MCP reconciliation.*persisted managed MCP configuration does not match/iu, + privateDetail: "registered MCP intent no longer matches", }, - "required host forwards could not be restored", - PRIVATE_RECOVERY_DETAIL, - ], - [ - "managed recovery failure", { - checked: true, - wasRunning: false, - recovered: false, - forwardRecovered: false, + agent, + label: "required-forward failure", + result: { + checked: true, + wasRunning: false, + recovered: true, + forwardRecovered: false, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: "the required API forward remained occupied", + } as StartupRecoveryResult, + expected: /host-forward recovery.*required host forward could not be restored/iu, + privateDetail: "the required API forward remained occupied", }, - "authenticated managed gateway recovery did not complete", - PRIVATE_RECOVERY_DETAIL, - ], - ] as const)("fails closed on %s without exposing raw recovery details (#8662)", async (_label, recovery, expected, privateDetail) => { - const h = harness(); - h.restoreStartupState.mockReturnValue(recovery); + { + agent, + label: "inconclusive managed result", + result: { + checked: true, + wasRunning: null, + recovered: false, + forwardRecovered: false, + } as StartupRecoveryResult, + expected: /inspection.*result was inconclusive/iu, + privateDetail: undefined, + }, + ]), + )("fails closed for $agent $label before final readiness verification (#8662)", async ({ + agent, + result, + expected, + privateDetail, + }) => { + const production = stubProductionStartupRecovery(result); + const h = harness({ restoreStartupState: undefined }); + h.getSandbox.mockReturnValue(sandbox({ agent })); + + const failure = await startSandbox("my-sandbox", h.deps).catch((error: unknown) => error); - let failure: unknown; - try { - await startSandbox("my-sandbox", h.deps); - } catch (error) { - failure = error; - } expect(failure).toBeInstanceOf(Error); - expect((failure as Error).message).toContain(expected); - expect((failure as Error).message).not.toContain(privateDetail); + expect((failure as Error).message).toMatch(expected); + expect((failure as Error).message).not.toContain(privateDetail ?? "private-detail-not-present"); + expect(production.check).toHaveBeenCalledOnce(); expect(h.verifyGateway).not.toHaveBeenCalled(); }); - it("accepts terminal agents without requiring gateway recovery (#8662)", () => { - expect(() => - assertSandboxStartupRecovery("terminal-box", { - checked: true, - wasRunning: null, - recovered: false, - forwardRecovered: false, - runtime: "terminal", - }), - ).not.toThrow(); - }); - it("reports the started container by name (#6026)", async () => { const h = harness(); @@ -317,19 +534,32 @@ describe("startSandbox", () => { it("still probes when the container was already running (#6026)", async () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([ - { name: "openshell-my-sandbox", status: "Up 5 minutes", running: true }, + { + containerId: ORIGINAL_CONTAINER_ID, + name: "openshell-my-sandbox", + status: "Up 5 minutes", + running: true, + }, ]); h.recoverDockerDriverSandbox.mockReturnValue({ recovered: true, via: "started-running-original", + containerId: ORIGINAL_CONTAINER_ID, containerName: "openshell-my-sandbox", }); const result = await startSandbox("my-sandbox", h.deps); expect(result.exitCode).toBe(0); - expect(h.restoreStartupState).toHaveBeenCalledWith("my-sandbox"); - expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); + expect(h.restoreStartupState).toHaveBeenCalledWith( + "my-sandbox", + expect.objectContaining({ expectedContainerId: ORIGINAL_CONTAINER_ID }), + ); + expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox", { + expectedContainerId: ORIGINAL_CONTAINER_ID, + preserveContainer: true, + requirePinnedManagedGatewayProof: true, + }); expect(h.restoreStartupState.mock.invocationCallOrder[0]).toBeLessThan( h.verifyGateway.mock.invocationCallOrder[0], ); @@ -341,6 +571,7 @@ describe("startSandbox", () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([ { + containerId: ORIGINAL_CONTAINER_ID, name: "openshell-my-sandbox", status: "Up 3 minutes (Paused)", running: true, @@ -350,13 +581,20 @@ describe("startSandbox", () => { const result = await startSandbox("my-sandbox", h.deps); expect(result.exitCode).toBe(0); - expect(h.dockerUnpause).toHaveBeenCalledWith("openshell-my-sandbox", { + expect(h.dockerUnpause).toHaveBeenCalledWith(ORIGINAL_CONTAINER_ID, { ignoreError: true, timeout: 30_000, }); expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); - expect(h.restoreStartupState).toHaveBeenCalledWith("my-sandbox"); - expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); + expect(h.restoreStartupState).toHaveBeenCalledWith( + "my-sandbox", + expect.objectContaining({ expectedContainerId: ORIGINAL_CONTAINER_ID }), + ); + expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox", { + expectedContainerId: ORIGINAL_CONTAINER_ID, + preserveContainer: true, + requirePinnedManagedGatewayProof: true, + }); expect(h.restoreStartupState.mock.invocationCallOrder[0]).toBeLessThan( h.verifyGateway.mock.invocationCallOrder[0], ); @@ -368,6 +606,7 @@ describe("startSandbox", () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([ { + containerId: ORIGINAL_CONTAINER_ID, name: "openshell-my-sandbox", status: "Up 3 minutes (Paused)", running: true, @@ -389,13 +628,34 @@ describe("startSandbox", () => { h.recoverDockerDriverSandbox.mockReturnValue({ recovered: true, via: "renamed-and-started-backup", + containerId: ORIGINAL_CONTAINER_ID, containerName: "openshell-my-sandbox", }); const result = await startSandbox("my-sandbox", h.deps); expect(result.exitCode).toBe(0); - expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); + expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox", { + expectedContainerId: ORIGINAL_CONTAINER_ID, + preserveContainer: true, + requirePinnedManagedGatewayProof: true, + }); + }); + + it("refuses startup recovery when Docker omits the immutable container identity (#8662)", async () => { + const h = harness(); + h.recoverDockerDriverSandbox.mockReturnValue({ + recovered: true, + via: "started-stopped-original", + containerName: "openshell-my-sandbox", + }); + + const result = await startSandbox("my-sandbox", h.deps); + + expect(result.exitCode).toBe(1); + expect(result.message).toMatch(/without its immutable container identity.*unpinned/iu); + expect(h.restoreStartupState).not.toHaveBeenCalled(); + expect(h.verifyGateway).not.toHaveBeenCalled(); }); it("names the Docker daemon outage instead of claiming the container was removed (#6026)", async () => { @@ -414,6 +674,26 @@ describe("startSandbox", () => { expect(h.verifyGateway).not.toHaveBeenCalled(); }); + it("returns actionable guidance when Docker discovery is malformed (#8662)", async () => { + const h = harness(); + h.findLabeledSandboxContainers.mockImplementation(() => { + throw new Error("opaque malformed discovery detail"); + }); + + const result = await startSandbox("my-sandbox", h.deps); + + expect(result).toEqual({ + exitCode: 1, + message: + " Docker could not verify the existing container for sandbox 'my-sandbox'. " + + "Run 'nemoclaw doctor', then retry 'nemoclaw my-sandbox start'.", + }); + expect(result.message).not.toContain("opaque malformed discovery detail"); + expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); + expect(h.restoreStartupState).not.toHaveBeenCalled(); + expect(h.verifyGateway).not.toHaveBeenCalled(); + }); + it("fails with the recovery detail and a rebuild hint when no container exists (#6026)", async () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([]); diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 49475fce8fc..56519c12a17 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -1,117 +1,235 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { CLI_NAME } from "../../cli/branding"; +import { cliName } from "../../onboard/branding"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, type RuntimeProviderBundleRegistry, } from "../../onboard/runtime-provider/access"; +import { SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED } from "../../shields"; import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; -import type { SandboxStartupRecoveryResult } from "./connect"; import { resolveSandboxLifecycleProvider, type SandboxLifecycleResult, } from "./runtime/lifecycle-runtime"; -function verifyGateway(sandboxName: string): Promise { - const { connectSandbox } = require("./connect") as { - connectSandbox: (name: string, options?: { probeOnly?: boolean }) => Promise; - }; - return connectSandbox(sandboxName, { probeOnly: true }); +export type SandboxStartGatewayVerificationOptions = { + expectedContainerId?: string; + preserveContainer?: boolean; + requirePinnedManagedGatewayProof?: boolean; +}; + +export const sandboxStartDependencies = { + loadConnect(): typeof import("./connect") { + return require("./connect") as typeof import("./connect"); + }, + loadShields(): typeof import("../../shields") { + return require("../../shields") as typeof import("../../shields"); + }, +}; + +async function verifyGateway( + sandboxName: string, + { + expectedContainerId, + preserveContainer = false, + requirePinnedManagedGatewayProof = false, + }: SandboxStartGatewayVerificationOptions = {}, +): Promise { + const { connectSandbox, pinnedManagedGatewayProbeFailure } = + sandboxStartDependencies.loadConnect(); + await connectSandbox(sandboxName, { + expectedContainerId, + preserveContainer, + probeOnly: true, + }); + if (!requirePinnedManagedGatewayProof || !expectedContainerId) return; + const failure = pinnedManagedGatewayProbeFailure(sandboxName, expectedContainerId); + if (!failure) return; + throw new Error( + `Sandbox '${sandboxName}' startup verification failed during final managed gateway health: ` + + `${sanitizedRecoveryDetail(`${failure.layer}: ${failure.detail}`)}. ` + + `The existing sandbox was preserved. Run \`${cliName()} ${sandboxName} recover\` and retry \`${cliName()} ${sandboxName} start\`.`, + ); +} + +type SandboxProcessRecoveryResult = import("./connect").SandboxStartupRecoveryResult; +type SandboxStartupRecoveryOptions = import("./connect").SandboxStartupRecoveryOptions; +type ManagedGatewayRecoveryFailure = + import("./connect").SandboxStartupManagedGatewayRecoveryFailure; + +type SandboxProcessRecoveryFailure = { + layer: + | "inspection" + | "managed gateway health" + | "managed supervisor" + | "OpenShell readiness" + | "secret boundary" + | "MCP reconciliation" + | "managed gateway recovery" + | "host-forward recovery"; + detail: string; +}; + +function sanitizedRecoveryDetail(value: unknown): string { + const raw = value instanceof Error && value.message ? value.message : String(value ?? ""); + return sandboxStartDependencies.loadConnect().sanitizeSandboxStartupRecoveryDetail(raw); +} + +function startupRecoveryFailure( + result: SandboxProcessRecoveryResult, + managedRecoveryFailure: ManagedGatewayRecoveryFailure | null = null, +): SandboxProcessRecoveryFailure | null { + if ("managedSupervisorUnavailable" in result && result.managedSupervisorUnavailable) { + return { + layer: "managed supervisor", + detail: sanitizedRecoveryDetail( + "managedSupervisorFailureDetail" in result + ? result.managedSupervisorFailureDetail + : "the authenticated supervisor did not become available in the existing container", + ), + }; + } + if ("managedGatewayProbeFailed" in result && result.managedGatewayProbeFailed) { + return { + layer: "managed gateway health", + detail: sanitizedRecoveryDetail( + "managedGatewayProbeFailureDetail" in result + ? result.managedGatewayProbeFailureDetail + : "the authenticated managed gateway probe did not pass", + ), + }; + } + if (!result.checked) { + return { + layer: "inspection", + detail: "the managed agent gateway could not be inspected after the container started", + }; + } + if ("secretBoundaryRefused" in result && result.secretBoundaryRefused) { + return { + layer: "secret boundary", + detail: "the agent secret boundary refused recovery", + }; + } + if ("mcpReconciliationRefused" in result && result.mcpReconciliationRefused) { + return { + layer: "MCP reconciliation", + detail: "the persisted managed MCP configuration does not match", + }; + } + if ("openshellReadinessFailed" in result && result.openshellReadinessFailed) { + return { + layer: "OpenShell readiness", + detail: sanitizedRecoveryDetail( + "openshellReadinessFailureDetail" in result + ? result.openshellReadinessFailureDetail + : "the existing sandbox did not become ready in OpenShell", + ), + }; + } + if ("forwardRecoveryFailed" in result && result.forwardRecoveryFailed) { + return { + layer: "host-forward recovery", + detail: "a required host forward could not be restored", + }; + } + if (result.wasRunning === false && result.recovered && !result.forwardRecovered) { + return { + layer: "host-forward recovery", + detail: "the primary dashboard/API host forward was not proven after gateway recovery", + }; + } + if (result.wasRunning === false && !result.recovered) { + return { + layer: "managed gateway recovery", + detail: sanitizedRecoveryDetail( + managedRecoveryFailure + ? `${managedRecoveryFailure.layer}: ${managedRecoveryFailure.detail}` + : "the managed agent gateway could not be restarted", + ), + }; + } + if (result.wasRunning === null && (!("runtime" in result) || result.runtime !== "terminal")) { + return { + layer: "inspection", + detail: "the managed agent gateway recovery result was inconclusive", + }; + } + return null; +} + +function assertStartupRecoverySucceeded( + sandboxName: string, + result: SandboxProcessRecoveryResult, + managedRecoveryFailure: ManagedGatewayRecoveryFailure | null = null, +): void { + const failure = startupRecoveryFailure(result, managedRecoveryFailure); + if (!failure) return; + throw new Error( + `Sandbox '${sandboxName}' started, but startup recovery failed during ${failure.layer}: ${failure.detail}. ` + + `The existing sandbox was preserved. Run \`${cliName()} ${sandboxName} recover\` and retry \`${cliName()} ${sandboxName} start\`.`, + ); } function restoreProcessState( sandboxName: string, - requireManagedRecovery: boolean, -): SandboxStartupRecoveryResult { - const { restoreSandboxStartupState } = require("./connect") as typeof import("./connect"); - return restoreSandboxStartupState(sandboxName, { requireManagedRecovery }); + options: SandboxStartupRecoveryOptions = {}, +): SandboxProcessRecoveryResult { + return sandboxStartDependencies.loadConnect().restoreSandboxStartupState(sandboxName, options); } -function restoreLockedStartupAccess(sandboxName: string): void { - const { restoreLockedStateDirStartupAccess } = - require("../../shields") as typeof import("../../shields"); - restoreLockedStateDirStartupAccess(sandboxName); +function restoreLockedStartupAccess(sandboxName: string, expectedContainerId?: string): void { + sandboxStartDependencies + .loadShields() + .restoreLockedStateDirStartupAccess(sandboxName, expectedContainerId); } export interface SandboxStartupStateDeps { agent?: SandboxEntry["agent"]; - restoreLockedStartupAccess?: (sandboxName: string) => void; - restoreProcessState?: typeof restoreProcessState; + processRecovery?: SandboxStartupRecoveryOptions; + restoreLockedStartupAccess?: (sandboxName: string, expectedContainerId?: string) => void; + restoreProcessState?: ( + sandboxName: string, + options?: SandboxStartupRecoveryOptions, + ) => SandboxProcessRecoveryResult; } function requiresManagedStartupRecovery(agent: SandboxEntry["agent"]): boolean { - const resolvedAgent = agent ?? "openclaw"; - return resolvedAgent === "openclaw" || resolvedAgent === "hermes"; + return !agent || agent === "openclaw" || agent === "hermes"; } export function restoreStoppedSandboxStartupState( sandboxName: string, deps: SandboxStartupStateDeps = {}, -): SandboxStartupRecoveryResult { - const agent = deps.agent ?? "openclaw"; - if (agent === "openclaw") { - (deps.restoreLockedStartupAccess ?? restoreLockedStartupAccess)(sandboxName); - } - return (deps.restoreProcessState ?? restoreProcessState)( - sandboxName, - requiresManagedStartupRecovery(agent), - ); -} - -function startupRecoveryFailure( - sandboxName: string, - detail: string, - nextStep = `Run '${CLI_NAME} ${sandboxName} recover' for classified repair guidance.`, -): Error { - return new Error(`Could not restore sandbox '${sandboxName}' after start: ${detail} ${nextStep}`); -} - -export function assertSandboxStartupRecovery( - sandboxName: string, - recovery: SandboxStartupRecoveryResult, -): void { - if ("runtime" in recovery && recovery.runtime === "terminal") return; - if (!recovery.checked) { - throw startupRecoveryFailure( +): SandboxProcessRecoveryResult { + const expectedContainerId = deps.processRecovery?.expectedContainerId; + if ((deps.agent ?? "openclaw") === "openclaw") { + (deps.restoreLockedStartupAccess ?? restoreLockedStartupAccess)( sandboxName, - "the managed gateway could not be inspected through the trusted container boundary.", - ); - } - if ("secretBoundaryRefused" in recovery && recovery.secretBoundaryRefused) { - throw startupRecoveryFailure( - sandboxName, - "the Hermes secret-boundary check refused the gateway state; Shields were left unchanged.", - ); - } - if ("mcpReconciliationRefused" in recovery && recovery.mcpReconciliationRefused) { - throw startupRecoveryFailure( - sandboxName, - "the Hermes MCP configuration does not match its persisted managed intent; Shields were left unchanged.", - ); - } - if ("forwardRecoveryFailed" in recovery && recovery.forwardRecoveryFailed) { - throw startupRecoveryFailure( - sandboxName, - "one or more required host forwards could not be restored.", - ); - } - if (!recovery.wasRunning && !recovery.recovered) { - throw startupRecoveryFailure( - sandboxName, - "authenticated managed gateway recovery did not complete.", - `Inspect '${CLI_NAME} ${sandboxName} logs', then run '${CLI_NAME} ${sandboxName} recover'.`, + expectedContainerId, ); } + const restore = deps.restoreProcessState ?? restoreProcessState; + return deps.processRecovery === undefined + ? restore(sandboxName) + : restore(sandboxName, deps.processRecovery); } export interface SandboxStartDeps { environment?: NodeJS.ProcessEnv; getSandbox?: typeof registry.getSandbox; runtimeProviders?: RuntimeProviderBundleRegistry; - restoreStartupState?: (sandboxName: string) => SandboxStartupRecoveryResult; - verifyGateway?: (sandboxName: string) => Promise; + startupRecovery?: SandboxStartupRecoveryOptions; + restoreStartupState?: ( + sandboxName: string, + options?: SandboxStartupRecoveryOptions, + ) => SandboxProcessRecoveryResult; + verifyGateway?: ( + sandboxName: string, + options?: SandboxStartGatewayVerificationOptions, + ) => Promise; log?: (message: string) => void; } @@ -144,21 +262,64 @@ export async function startSandbox( if (preflight) return preflight; const result = resolved.lifecycle.start(input); if (result.exitCode !== 0) return result; + const expectedContainerId = result.runtimeHandle; + const pinnedManagedStartupRecovery = + requiresManagedStartupRecovery(resolved.sandbox.agent) && Boolean(expectedContainerId); await resolved.lifecycle.verifyStarted(input, async (name) => { log(" Restoring sandbox startup state…"); - const restoreStartupState = - deps.restoreStartupState ?? - ((sandboxNameToRestore: string) => - restoreStoppedSandboxStartupState(sandboxNameToRestore, { - agent: resolved.sandbox.agent, - })); - const recovery = restoreStartupState(name); - if (requiresManagedStartupRecovery(resolved.sandbox.agent)) { - assertSandboxStartupRecovery(name, recovery); + const recoveryFailureState: { current: ManagedGatewayRecoveryFailure | null } = { + current: null, + }; + const processRecovery: SandboxStartupRecoveryOptions | undefined = pinnedManagedStartupRecovery + ? { + ...deps.startupRecovery, + expectedContainerId, + onRecoveryFailure: (failure) => { + recoveryFailureState.current = failure; + deps.startupRecovery?.onRecoveryFailure?.(failure); + }, + } + : undefined; + let recovery: SandboxProcessRecoveryResult; + try { + recovery = deps.restoreStartupState + ? processRecovery + ? deps.restoreStartupState(name, processRecovery) + : deps.restoreStartupState(name) + : restoreStoppedSandboxStartupState(name, { + agent: resolved.sandbox.agent, + processRecovery, + }); + } catch (error) { + if ( + error && + typeof error === "object" && + "code" in error && + error.code === SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED + ) { + throw new Error( + `Sandbox '${name}' started, but its expired Shields auto-restore must finish before startup recovery. ` + + `The existing sandbox was preserved. Run \`${cliName()} ${name} shields up\`, then retry \`${cliName()} ${name} start\`.`, + ); + } + throw new Error( + `Sandbox '${name}' started, but startup state restoration failed: ${sanitizedRecoveryDetail(error)}. ` + + `The existing sandbox was preserved. Run \`${cliName()} ${name} recover\` and retry \`${cliName()} ${name} start\`.`, + ); + } + if (pinnedManagedStartupRecovery) { + assertStartupRecoverySucceeded(name, recovery, recoveryFailureState.current); } log(" Checking gateway health and host forwards…"); - await (deps.verifyGateway ?? verifyGateway)(name); + const gatewayVerification = deps.verifyGateway ?? verifyGateway; + await (pinnedManagedStartupRecovery + ? gatewayVerification(name, { + expectedContainerId, + preserveContainer: true, + requirePinnedManagedGatewayProof: true, + }) + : gatewayVerification(name)); }); return { exitCode: 0 }; } diff --git a/src/lib/actions/sandbox/stop.test.ts b/src/lib/actions/sandbox/stop.test.ts index 4f665a01800..6b57bd29b93 100644 --- a/src/lib/actions/sandbox/stop.test.ts +++ b/src/lib/actions/sandbox/stop.test.ts @@ -17,8 +17,9 @@ function sandbox(values: Partial = {}): SandboxEntry { return { name: "my-sandbox", ...values }; } -function container(name: string, running: boolean) { +function container(name: string, running: boolean, containerId = "a".repeat(64)) { return { + containerId, name, status: running ? "Up 5 minutes" : "Exited (0) 2 hours ago", running, @@ -175,7 +176,7 @@ describe("stopSandbox", () => { warn: expect.any(Function), }), ); - expect(h.dockerStop).toHaveBeenCalledWith("openshell-my-sandbox", { + expect(h.dockerStop).toHaveBeenCalledWith("a".repeat(64), { ignoreError: true, timeout: 30_000, }); @@ -275,6 +276,7 @@ describe("stopSandbox", () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([ { + containerId: "a".repeat(64), name: "openshell-my-sandbox", status: "Restarting (137) 2 seconds ago", running: false, @@ -284,7 +286,7 @@ describe("stopSandbox", () => { const result = stopSandbox("my-sandbox", h.deps); expect(result.exitCode).toBe(0); - expect(h.dockerStop).toHaveBeenCalledWith("openshell-my-sandbox", { + expect(h.dockerStop).toHaveBeenCalledWith("a".repeat(64), { ignoreError: true, timeout: 30_000, }); @@ -294,6 +296,7 @@ describe("stopSandbox", () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([ { + containerId: "a".repeat(64), name: "openshell-my-sandbox", status: "Up 5 minutes (Paused)", running: true, @@ -309,14 +312,17 @@ describe("stopSandbox", () => { it("stops every running labeled container, including backup siblings (#6026)", () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([ - container("openshell-my-sandbox", true), - container("openshell-my-sandbox-nemoclaw-gpu-backup-1700000000000", true), + container("openshell-my-sandbox", true, "a".repeat(64)), + container("openshell-my-sandbox-nemoclaw-gpu-backup-1700000000000", true, "b".repeat(64)), ]); const result = stopSandbox("my-sandbox", h.deps); expect(result.exitCode).toBe(0); - expect(h.dockerStop).toHaveBeenCalledTimes(2); + expect(h.dockerStop.mock.calls.map(([containerId]) => containerId)).toEqual([ + "a".repeat(64), + "b".repeat(64), + ]); }); it("continues to docker stop when the graceful channel stop throws (#6026)", () => { @@ -348,6 +354,27 @@ describe("stopSandbox", () => { expect(h.dockerStop).not.toHaveBeenCalled(); }); + it("returns actionable guidance without stopping channels when Docker discovery is malformed (#8662)", () => { + const h = harness({ + findLabeledSandboxContainers: vi.fn(() => { + throw new Error("opaque malformed discovery detail"); + }), + }); + + const result = stopSandbox("my-sandbox", h.deps); + + expect(result).toEqual({ + exitCode: 1, + message: + " Docker could not verify the existing container for sandbox 'my-sandbox'. " + + "Run 'nemoclaw doctor', then retry 'nemoclaw my-sandbox stop'.", + }); + expect(result.message).not.toContain("opaque malformed discovery detail"); + expect(h.stopSandboxChannels).not.toHaveBeenCalled(); + expect(h.dockerStop).not.toHaveBeenCalled(); + expect(h.teardownSandboxDashboardForward).not.toHaveBeenCalled(); + }); + it("fails with a rebuild hint when no labeled container exists (#6026)", () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([]); @@ -431,8 +458,8 @@ describe("stopSandbox", () => { it("attempts every container and aggregates failures when one stop fails (#6026)", () => { const h = harness(); h.findLabeledSandboxContainers.mockReturnValue([ - container("openshell-my-sandbox", true), - container("openshell-my-sandbox-nemoclaw-gpu-backup-1700000000000", true), + container("openshell-my-sandbox", true, "a".repeat(64)), + container("openshell-my-sandbox-nemoclaw-gpu-backup-1700000000000", true, "b".repeat(64)), ]); // First container fails to stop; the sibling still must be attempted. h.dockerStop.mockReturnValueOnce({ status: 137 }).mockReturnValueOnce({ status: 0 }); @@ -442,14 +469,10 @@ describe("stopSandbox", () => { expect(result.exitCode).toBe(1); expect(h.dockerStop).toHaveBeenCalledTimes(2); expect(h.teardownSandboxDashboardForward).not.toHaveBeenCalled(); - expect(h.dockerStop).toHaveBeenNthCalledWith( - 2, - "openshell-my-sandbox-nemoclaw-gpu-backup-1700000000000", - { - ignoreError: true, - timeout: 30_000, - }, - ); + expect(h.dockerStop).toHaveBeenNthCalledWith(2, "b".repeat(64), { + ignoreError: true, + timeout: 30_000, + }); expect(result.message).toContain("openshell-my-sandbox"); expect(result.message).toContain("137"); expect(result.message).not.toContain("gpu-backup"); @@ -463,7 +486,7 @@ describe("stopSandbox", () => { // The deps surface has no removal lever at all; assert the only docker // mutation issued is the stop of the labeled container. expect(h.dockerStop.mock.calls).toEqual([ - ["openshell-my-sandbox", { ignoreError: true, timeout: 30_000 }], + ["a".repeat(64), { ignoreError: true, timeout: 30_000 }], ]); }); }); diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts index 53c7eca7fe7..3559ad87250 100644 --- a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts +++ b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts @@ -12,6 +12,29 @@ interface FakeRunResult { status: number; } +const ORIGINAL_CONTAINER_ID = "a".repeat(64); +const BACKUP_CONTAINER_ID = "b".repeat(64); +const NEWEST_BACKUP_CONTAINER_ID = "c".repeat(64); + +function containerIdForFixtureName(name: string): string { + return name.endsWith("1717290000000") + ? NEWEST_BACKUP_CONTAINER_ID + : name.includes("-nemoclaw-gpu-backup-") + ? BACKUP_CONTAINER_ID + : ORIGINAL_CONTAINER_ID; +} + +function withFullContainerIds(output: string): string { + return output + .split(/\r?\n/u) + .map((line) => { + const trimmed = line.trim(); + const [name, ...status] = trimmed.split("\t"); + return trimmed ? `${containerIdForFixtureName(name)}\t${name}\t${status.join("\t")}` : ""; + }) + .join("\n"); +} + function fakeStart(status = 0): (name: string, opts?: Record) => FakeRunResult { return () => ({ status }); } @@ -25,39 +48,52 @@ function fakeRename( function fakeCapture( output: string, inspectOutputs: string[] = ["running\thealthy"], - onInspect: (opts?: Record) => void = () => undefined, + onInspect: (args: readonly string[], opts?: Record) => void = () => undefined, ): (args: readonly string[], opts?: Record) => string { let inspectIndex = 0; return (args, opts) => { switch (args[0]) { case "inspect": { - onInspect(opts); + onInspect(args, opts); const index = Math.min(inspectIndex, inspectOutputs.length - 1); inspectIndex += 1; return inspectOutputs[index] ?? ""; } default: - return output; + return withFullContainerIds(output); } }; } describe("findLabeledSandboxContainers", () => { it("parses the OpenShell-labeled container list and detects running state", () => { - const containers = findLabeledSandboxContainers("e2e-x", { - dockerCapture: fakeCapture( + const dockerCapture = vi.fn( + fakeCapture( "openshell-e2e-x\tUp 2 hours\n" + "openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited (0) 10 minutes ago\n", ), + ); + const containers = findLabeledSandboxContainers("e2e-x", { + dockerCapture, }); expect(containers).toEqual([ - { name: "openshell-e2e-x", status: "Up 2 hours", running: true }, { + containerId: ORIGINAL_CONTAINER_ID, + name: "openshell-e2e-x", + status: "Up 2 hours", + running: true, + }, + { + containerId: BACKUP_CONTAINER_ID, name: "openshell-e2e-x-nemoclaw-gpu-backup-1717280000000", status: "Exited (0) 10 minutes ago", running: false, }, ]); + expect(dockerCapture).toHaveBeenCalledWith( + expect.arrayContaining(["--no-trunc", "{{.ID}}\t{{.Names}}\t{{.Status}}"]), + expect.anything(), + ); }); it("returns an empty array when docker ps has no labeled rows", () => { @@ -68,7 +104,27 @@ describe("findLabeledSandboxContainers", () => { const containers = findLabeledSandboxContainers("e2e-x", { dockerCapture: fakeCapture("\n openshell-e2e-x\tCreated\n\n"), }); - expect(containers).toEqual([{ name: "openshell-e2e-x", status: "Created", running: false }]); + expect(containers).toEqual([ + { + containerId: ORIGINAL_CONTAINER_ID, + name: "openshell-e2e-x", + status: "Created", + running: false, + }, + ]); + }); + + it.each([ + ["truncated identity", `abc123\topenshell-e2e-x\tCreated\n`], + ["uppercase identity", `${"A".repeat(64)}\topenshell-e2e-x\tCreated\n`], + ["control character in name", `${ORIGINAL_CONTAINER_ID}\topenshell-e2e-\u007fx\tCreated\n`], + ["empty status", `${ORIGINAL_CONTAINER_ID}\topenshell-e2e-x\t\n`], + ])("rejects malformed metadata with a %s", (_label, row) => { + expect(() => + findLabeledSandboxContainers("e2e-x", { + dockerCapture: () => row, + }), + ).toThrow(/malformed OpenShell sandbox container metadata/u); }); }); @@ -76,21 +132,25 @@ describe("recoverDockerDriverSandbox — running original (no-op)", () => { it("waits for an already-running container to become healthy without starting it", () => { const start = vi.fn(fakeStart(0)); const sleep = vi.fn(); + const inspectTargets: string[] = []; const result = recoverDockerDriverSandbox("e2e-x", { - dockerCapture: fakeCapture("openshell-e2e-x\tUp 5 seconds\n", [ - "running\tstarting", - "running\thealthy", - ]), + dockerCapture: fakeCapture( + "openshell-e2e-x\tUp 5 seconds\n", + ["running\tstarting", "running\thealthy"], + (args) => inspectTargets.push(String(args.at(-1))), + ), dockerStart: start, sleep, }); expect(result).toEqual({ recovered: true, via: "started-running-original", + containerId: ORIGINAL_CONTAINER_ID, containerName: "openshell-e2e-x", }); expect(start).not.toHaveBeenCalled(); expect(sleep).toHaveBeenCalledOnce(); + expect(inspectTargets).toEqual([ORIGINAL_CONTAINER_ID, ORIGINAL_CONTAINER_ID]); }); }); @@ -106,11 +166,12 @@ describe("recoverDockerDriverSandbox — paused original (unpause)", () => { expect(result).toEqual({ recovered: true, via: "unpaused-original", + containerId: ORIGINAL_CONTAINER_ID, containerName: "openshell-e2e-x", }); expect(unpause).toHaveBeenCalledTimes(1); expect(unpause).toHaveBeenCalledWith( - "openshell-e2e-x", + ORIGINAL_CONTAINER_ID, expect.objectContaining({ ignoreError: true }), ); expect(start).not.toHaveBeenCalled(); @@ -139,6 +200,7 @@ describe("recoverDockerDriverSandbox — paused original (unpause)", () => { expect(result).toMatchObject({ recovered: false, via: null, + containerId: ORIGINAL_CONTAINER_ID, containerName: "openshell-e2e-x", detail: expect.stringContaining("runtime=dead, health=unhealthy"), }); @@ -155,11 +217,12 @@ describe("recoverDockerDriverSandbox — stopped original (start)", () => { expect(result).toEqual({ recovered: true, via: "started-stopped-original", + containerId: ORIGINAL_CONTAINER_ID, containerName: "openshell-e2e-x", }); expect(start).toHaveBeenCalledTimes(1); expect(start).toHaveBeenCalledWith( - "openshell-e2e-x", + ORIGINAL_CONTAINER_ID, expect.objectContaining({ ignoreError: true }), ); }); @@ -189,6 +252,7 @@ describe("recoverDockerDriverSandbox — stopped original (start)", () => { expect(result).toEqual({ recovered: true, via: "started-stopped-original", + containerId: ORIGINAL_CONTAINER_ID, containerName: "openshell-e2e-x", }); expect(sleep).toHaveBeenCalledTimes(2); @@ -209,6 +273,7 @@ describe("recoverDockerDriverSandbox — stopped original (start)", () => { expect(result).toEqual({ recovered: true, via: "started-stopped-original", + containerId: ORIGINAL_CONTAINER_ID, containerName: "openshell-e2e-x", }); expect(sleep).not.toHaveBeenCalled(); @@ -227,6 +292,7 @@ describe("recoverDockerDriverSandbox — stopped original (start)", () => { expect(result).toEqual({ recovered: true, via: "started-stopped-original", + containerId: ORIGINAL_CONTAINER_ID, containerName: "openshell-e2e-x", }); expect(sleep).not.toHaveBeenCalled(); @@ -249,6 +315,7 @@ describe("recoverDockerDriverSandbox — stopped original (start)", () => { expect(result).toEqual({ recovered: false, via: null, + containerId: ORIGINAL_CONTAINER_ID, containerName: "openshell-e2e-x", detail: "docker container openshell-e2e-x did not become ready after recovery " + @@ -264,7 +331,7 @@ describe("recoverDockerDriverSandbox — stopped original (start)", () => { const capture = fakeCapture( "openshell-e2e-x\tExited (137) 30 seconds ago\n", ["running\tstarting"], - (opts) => { + (_args, opts) => { inspectStartTimes.push(currentMs); const timeout = Number(opts?.timeout); inspectTimeouts.push(timeout); @@ -315,9 +382,12 @@ describe("recoverDockerDriverSandbox — backup-only (rename + start)", () => { it("renames the backup sibling back to the original name and starts it", () => { const rename = vi.fn(fakeRename(0)); const start = vi.fn(fakeStart(0)); + const inspectTargets: string[] = []; const result = recoverDockerDriverSandbox("e2e-x", { dockerCapture: fakeCapture( "openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited (0) 5 minutes ago\n", + ["running\thealthy"], + (args) => inspectTargets.push(String(args.at(-1))), ), dockerRename: rename, dockerStart: start, @@ -325,17 +395,19 @@ describe("recoverDockerDriverSandbox — backup-only (rename + start)", () => { expect(result).toEqual({ recovered: true, via: "renamed-and-started-backup", + containerId: BACKUP_CONTAINER_ID, containerName: "openshell-e2e-x", }); expect(rename).toHaveBeenCalledWith( - "openshell-e2e-x-nemoclaw-gpu-backup-1717280000000", + BACKUP_CONTAINER_ID, "openshell-e2e-x", expect.objectContaining({ ignoreError: true }), ); expect(start).toHaveBeenCalledWith( - "openshell-e2e-x", + BACKUP_CONTAINER_ID, expect.objectContaining({ ignoreError: true }), ); + expect(inspectTargets).toEqual([BACKUP_CONTAINER_ID]); }); it("hands a renamed backup to lifecycle verification while Docker health is starting (#8112)", () => { @@ -354,6 +426,7 @@ describe("recoverDockerDriverSandbox — backup-only (rename + start)", () => { expect(result).toEqual({ recovered: true, via: "renamed-and-started-backup", + containerId: BACKUP_CONTAINER_ID, containerName: "openshell-e2e-x", }); expect(sleep).not.toHaveBeenCalled(); @@ -372,7 +445,7 @@ describe("recoverDockerDriverSandbox — backup-only (rename + start)", () => { }); expect(rename).toHaveBeenCalledTimes(1); expect(rename).toHaveBeenCalledWith( - "openshell-e2e-x-nemoclaw-gpu-backup-1717290000000", + NEWEST_BACKUP_CONTAINER_ID, "openshell-e2e-x", expect.anything(), ); @@ -411,6 +484,7 @@ describe("recoverDockerDriverSandbox — backup-only (rename + start)", () => { expect(result).toMatchObject({ recovered: false, via: null, + containerId: BACKUP_CONTAINER_ID, containerName: "openshell-e2e-x", detail: expect.stringContaining("runtime=removing, health=none"), }); @@ -431,7 +505,7 @@ describe("recoverDockerDriverSandbox — collision and missing cases", () => { }); expect(result.via).toBe("started-stopped-original"); expect(rename).not.toHaveBeenCalled(); - expect(start).toHaveBeenCalledWith("openshell-e2e-x", expect.anything()); + expect(start).toHaveBeenCalledWith(ORIGINAL_CONTAINER_ID, expect.anything()); }); it("returns recovered=false when no labeled container exists at all", () => { diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.ts b/src/lib/onboard/docker-driver-sandbox-recovery.ts index 71c6c49be1a..be425ab630d 100644 --- a/src/lib/onboard/docker-driver-sandbox-recovery.ts +++ b/src/lib/onboard/docker-driver-sandbox-recovery.ts @@ -74,6 +74,7 @@ const DOCKER_RECOVERY_READY_POLL_INTERVAL_MS = 1_000; const DOCKER_RECOVERY_READY_MAX_ATTEMPTS = Math.floor(DOCKER_RECOVERY_READY_TIMEOUT_MS / DOCKER_RECOVERY_READY_POLL_INTERVAL_MS) + 1; const MAX_DOCKER_CONTAINER_NAME_LENGTH = 253; +const FULL_DOCKER_CONTAINER_ID_PATTERN = /^[0-9a-f]{64}$/u; const DOCKER_RECOVERY_SLEEP_BUFFER = new Int32Array(new SharedArrayBuffer(4)); function sleepForDockerRecovery(ms: number): void { @@ -96,6 +97,8 @@ export type DockerDriverRecoveryVia = export interface DockerDriverRecoveryResult { recovered: boolean; via: DockerDriverRecoveryVia | null; + /** Immutable full Docker ID of the container recovery acted on. */ + containerId?: string; /** Container name (post-rename if applicable) the recovery acted on. */ containerName?: string; /** @@ -108,10 +111,10 @@ export interface DockerDriverRecoveryResult { export interface DockerDriverRecoveryDeps { /** `docker ps -a --filter ... --format ...` runner. */ dockerCapture?: (args: readonly string[], opts?: Record) => string; - dockerStart?: (name: string, opts?: Record) => { status?: number | null }; - dockerUnpause?: (name: string, opts?: Record) => { status?: number | null }; + dockerStart?: (target: string, opts?: Record) => { status?: number | null }; + dockerUnpause?: (target: string, opts?: Record) => { status?: number | null }; dockerRename?: ( - oldName: string, + target: string, newName: string, opts?: Record, ) => { status?: number | null }; @@ -129,6 +132,7 @@ export interface DockerDriverRecoveryDeps { } interface LabeledContainer { + containerId: string; name: string; status: string; /** True when the container is in a Docker `running` state. */ @@ -168,8 +172,8 @@ function originalNameFromBackup(backupName: string): string { * name across all states (running, exited, paused, dead). Empty array * means recovery has nothing to act on. * - * Output rows are tab-separated `\t` from - * `--format '{{.Names}}\t{{.Status}}'`. `Status` strings start with + * Output rows are tab-separated `\t\t` from + * `--no-trunc --format '{{.ID}}\t{{.Names}}\t{{.Status}}'`. `Status` strings start with * `Up` for running containers (e.g. `Up 5 minutes`) and `Exited` / * `Created` / `Paused` / `Dead` otherwise. */ @@ -182,12 +186,13 @@ export function findLabeledSandboxContainers( [ "ps", "-a", + "--no-trunc", "--filter", `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, "--filter", `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, "--format", - "{{.Names}}\t{{.Status}}", + "{{.ID}}\t{{.Names}}\t{{.Status}}", ], { ignoreError: true, timeout: DOCKER_PROBE_TIMEOUT_MS }, ); @@ -196,9 +201,17 @@ export function findLabeledSandboxContainers( .map((line) => line.trim()) .filter(Boolean) .map((line) => { - const [name, ...rest] = line.split("\t"); + const [containerId, name, ...rest] = line.split("\t"); const status = rest.join("\t").trim(); - return { name, status, running: status.startsWith("Up") }; + if ( + !FULL_DOCKER_CONTAINER_ID_PATTERN.test(containerId) || + !name || + !status || + /[\u0000-\u001f\u007f]/u.test(name) + ) { + throw new Error("Docker returned malformed OpenShell sandbox container metadata."); + } + return { containerId, name, status, running: status.startsWith("Up") }; }); } @@ -247,7 +260,7 @@ interface ContainerReadinessResult { * Docker's own readiness contract settles. */ function waitForRecoveredContainerReady( - containerName: string, + containerId: string, deps: ReturnType, readiness: NonNullable, ): ContainerReadinessResult { @@ -263,7 +276,7 @@ function waitForRecoveredContainerReady( "container", "--format", "{{.State.Status}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}", - containerName, + containerId, ], { ignoreError: true, timeout: Math.min(DOCKER_PROBE_TIMEOUT_MS, remainingMs) }, ); @@ -293,15 +306,17 @@ function waitForRecoveredContainerReady( function recoveredAfterReadiness( containerName: string, + containerId: string, via: DockerDriverRecoveryVia, deps: ReturnType, readiness: NonNullable, ): DockerDriverRecoveryResult { - const result = waitForRecoveredContainerReady(containerName, deps, readiness); + const result = waitForRecoveredContainerReady(containerId, deps, readiness); if (!result.ready) { return { recovered: false, via: null, + containerId, containerName, detail: `docker container ${containerName} did not become ready after recovery ` + @@ -311,6 +326,7 @@ function recoveredAfterReadiness( return { recovered: true, via, + containerId, containerName, }; } @@ -322,9 +338,9 @@ function recoveredAfterReadiness( * * Order of operations: * 1. Already-running labeled container → no-op success. - * 2. Stopped labeled original → `docker start `. + * 2. Stopped labeled original → `docker start `. * 3. Backup-only (`*-nemoclaw-gpu-backup-*` sibling) → `docker - * rename ` then `docker start `. + * rename ` then `docker start `. * 4. Conflict (backup AND a stale stopped original with the same * base name) → start the original; leave the backup alone. * 5. No labeled container at all → `{ recovered: false }`. @@ -348,7 +364,7 @@ export function recoverDockerDriverSandbox( if (runningOriginal) { if (isPausedStatus(runningOriginal.status)) { - const unpause = d.dockerUnpause(runningOriginal.name, { + const unpause = d.dockerUnpause(runningOriginal.containerId, { ignoreError: true, timeout: DOCKER_OPERATION_TIMEOUT_MS, }); @@ -356,23 +372,37 @@ export function recoverDockerDriverSandbox( return { recovered: false, via: null, + containerId: runningOriginal.containerId, containerName: runningOriginal.name, detail: `docker unpause ${runningOriginal.name} failed (exit ${unpause.status ?? "unknown"}).`, }; } - return recoveredAfterReadiness(runningOriginal.name, "unpaused-original", d, readiness); + return recoveredAfterReadiness( + runningOriginal.name, + runningOriginal.containerId, + "unpaused-original", + d, + readiness, + ); } - return recoveredAfterReadiness(runningOriginal.name, "started-running-original", d, readiness); + return recoveredAfterReadiness( + runningOriginal.name, + runningOriginal.containerId, + "started-running-original", + d, + readiness, + ); } if (stoppedOriginal) { - const result = d.dockerStart(stoppedOriginal.name, { + const result = d.dockerStart(stoppedOriginal.containerId, { ignoreError: true, timeout: DOCKER_OPERATION_TIMEOUT_MS, }); if (result.status === 0) { return recoveredAfterReadiness( stoppedOriginal.name, + stoppedOriginal.containerId, "started-stopped-original", d, readiness, @@ -381,13 +411,15 @@ export function recoverDockerDriverSandbox( return { recovered: false, via: null, + containerId: stoppedOriginal.containerId, + containerName: stoppedOriginal.name, detail: `docker start ${stoppedOriginal.name} failed (exit ${result.status ?? "unknown"})`, }; } if (backup) { const restoreName = buildBackupRestoreName(originalNameFromBackup(backup.name)); - const renameResult = d.dockerRename(backup.name, restoreName, { + const renameResult = d.dockerRename(backup.containerId, restoreName, { ignoreError: true, timeout: DOCKER_OPERATION_TIMEOUT_MS, }); @@ -395,19 +427,29 @@ export function recoverDockerDriverSandbox( return { recovered: false, via: null, + containerId: backup.containerId, + containerName: backup.name, detail: `docker rename ${backup.name} ${restoreName} failed (exit ${renameResult.status ?? "unknown"})`, }; } - const startResult = d.dockerStart(restoreName, { + const startResult = d.dockerStart(backup.containerId, { ignoreError: true, timeout: DOCKER_OPERATION_TIMEOUT_MS, }); if (startResult.status === 0) { - return recoveredAfterReadiness(restoreName, "renamed-and-started-backup", d, readiness); + return recoveredAfterReadiness( + restoreName, + backup.containerId, + "renamed-and-started-backup", + d, + readiness, + ); } return { recovered: false, via: null, + containerId: backup.containerId, + containerName: restoreName, detail: `docker start ${restoreName} after backup rename failed (exit ${startResult.status ?? "unknown"})`, }; } diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index d2f41f873b2..d251abd1853 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -122,6 +122,8 @@ export interface RuntimeProviderLifecycleInput { export type RuntimeProviderLifecycleResult = { readonly exitCode: number; readonly message?: string; + /** Opaque immutable runtime identity captured by a successful direct start. */ + readonly runtimeHandle?: string; }; export type RuntimeProviderLifecycleStopOutcome = RuntimeProviderLifecycleResult & { diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index 5863bdcaee7..038282f9f06 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -139,6 +139,18 @@ function isAtRestStatus(status: string): boolean { return AT_REST_STATUS_PREFIXES.some((prefix) => status.startsWith(prefix)); } +function dockerLifecycleInspectionFailure( + action: "start" | "stop", + input: RuntimeProviderLifecycleInput, +): RuntimeProviderLifecycleResult { + return { + exitCode: 1, + message: + ` Docker could not verify the existing container for sandbox '${input.sandboxName}'. ` + + `Run '${cliName()} doctor', then retry '${cliName()} ${input.sandboxName} ${action}'.`, + }; +} + function startDockerSandbox( input: RuntimeProviderLifecycleInput, deps: DockerRuntimeProviderDependencies, @@ -146,7 +158,7 @@ function startDockerSandbox( const containers = deps.findLabeledSandboxContainers(input.sandboxName); const paused = containers.find((container) => isPausedStatus(container.status)); if (paused) { - const result = deps.unpauseContainer(paused.name, { + const result = deps.unpauseContainer(paused.containerId, { ignoreError: true, timeout: DOCKER_OPERATION_TIMEOUT_MS, }); @@ -157,7 +169,7 @@ function startDockerSandbox( }; } input.log(` Container '${paused.name}' unpaused.`); - return { exitCode: 0 }; + return { exitCode: 0, runtimeHandle: paused.containerId }; } // Docker health is an image-level signal, not the lifecycle authority for @@ -175,12 +187,20 @@ function startDockerSandbox( `If the container was removed, run '${cliName()} ${input.sandboxName} rebuild' to recreate it.`, }; } + if (!recovery.containerId) { + return { + exitCode: 1, + message: + ` Docker reported sandbox '${input.sandboxName}' running without its immutable ` + + "container identity; refusing startup recovery against an unpinned container.", + }; + } if (recovery.via === "started-running-original") { input.log(` Sandbox '${input.sandboxName}' is already running.`); } else { input.log(` Container '${recovery.containerName ?? input.sandboxName}' started.`); } - return { exitCode: 0 }; + return { exitCode: 0, runtimeHandle: recovery.containerId }; } function stopDockerSandbox( @@ -205,7 +225,7 @@ function stopDockerSandbox( const failures: string[] = []; for (const container of stoppable) { input.log(` Stopping container '${container.name}'…`); - const result = deps.stopContainer(container.name, { + const result = deps.stopContainer(container.containerId, { ignoreError: true, timeout: DOCKER_OPERATION_TIMEOUT_MS, }); @@ -349,9 +369,21 @@ export function createDockerRuntimeProviderBundle( providerId, supported: true, channelStopTransport: "docker-kubectl-first", - start: (input) => startDockerSandbox(input, deps), + start: (input) => { + try { + return startDockerSandbox(input, deps); + } catch { + return dockerLifecycleInspectionFailure("start", input); + } + }, verifyStarted: (input, verifyGateway) => verifyGateway(input.sandboxName), - stop: (input, hooks) => stopDockerSandbox(input, hooks, deps), + stop: (input, hooks) => { + try { + return stopDockerSandbox(input, hooks, deps); + } catch { + return dockerLifecycleInspectionFailure("stop", input); + } + }, }, mutationAuthority: { providerId, diff --git a/src/lib/onboard/runtime-provider/podman.test.ts b/src/lib/onboard/runtime-provider/podman.test.ts index 0fa5c627211..6f51904f4f5 100644 --- a/src/lib/onboard/runtime-provider/podman.test.ts +++ b/src/lib/onboard/runtime-provider/podman.test.ts @@ -146,7 +146,12 @@ describe("dormant Podman runtime provider", () => { )("runs basic CPU start and stop for %s through an injected bundle", async (agent) => { const runtime = providerHarness(agent); const verifyGateway = vi.fn(async () => undefined); - const restoreStartupState = vi.fn(); + const restoreStartupState = vi.fn(() => ({ + checked: true as const, + wasRunning: true, + recovered: false, + forwardRecovered: false, + })); const stopSandboxChannels = vi.fn(); await expect( @@ -188,7 +193,12 @@ describe("dormant Podman runtime provider", () => { startSandbox(runtime.sandboxName, { getSandbox: () => runtime.entry, runtimeProviders: runtime.providers, - restoreStartupState: vi.fn(), + restoreStartupState: vi.fn(() => ({ + checked: true as const, + wasRunning: true, + recovered: false, + forwardRecovered: false, + })), verifyGateway, log: vi.fn(), }), diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index 99fdf735d16..6c569b8dc3c 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -1106,6 +1106,19 @@ function isDurableContainmentFailure(error: unknown): boolean { ); } +export const SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED = "NEMOCLAW_SHIELDS_AUTO_RESTORE_REQUIRED"; + +class ShieldsStartupAutoRestoreRequiredError extends Error { + readonly code = SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED; + + constructor() { + super( + "Expired Shields auto-restore must complete before startup access can be repaired in the pinned container", + ); + this.name = "ShieldsStartupAutoRestoreRequiredError"; + } +} + function retryInlineAutoRestore( sandboxName: string, marker: TimerMarker & { processToken: string }, @@ -1163,11 +1176,29 @@ function withExpiredAutoRestoreDeadlineFence( sandboxName: string, command: string, operation: (allowInlineRecovery: boolean) => T, + { allowInlineRecovery = true }: { allowInlineRecovery?: boolean } = {}, ): T { - const expiredMarker = inspectExpiredAutoRestoreMarker(sandboxName); - const takeover = inspectExpiredAutoRestoreTakeover(sandboxName, expiredMarker); const runWithHostLock = (callback: () => T) => withTimerBoundShieldsMutationLock(sandboxName, command, callback); + if (!allowInlineRecovery) { + const assertNoExpiredAutoRestore = () => { + if (inspectExpiredAutoRestoreMarker(sandboxName)) { + throw new ShieldsStartupAutoRestoreRequiredError(); + } + }; + assertNoExpiredAutoRestore(); + const runWithoutInlineRecovery = () => + runWithHostLock(() => { + assertNoExpiredAutoRestore(); + return operation(false); + }); + if (isMcpLifecycleLockHeld(sandboxName, STATE_DIR)) return runWithoutInlineRecovery(); + return withMcpLifecycleLockSync(sandboxName, runWithoutInlineRecovery, { + stateDir: STATE_DIR, + }); + } + const expiredMarker = inspectExpiredAutoRestoreMarker(sandboxName); + const takeover = inspectExpiredAutoRestoreTakeover(sandboxName, expiredMarker); const recoverThenRun = () => withTimerBoundAutoRestoreLock(sandboxName, command, () => { if (takeover) retryInlineAutoRestore(sandboxName, takeover.marker); @@ -1381,11 +1412,11 @@ function isShieldsState(value: unknown): value is ShieldsState { // file stays focused on shields state transitions. // --------------------------------------------------------------------------- -function stateDirLockExec(sandboxName: string) { +function stateDirLockExec(sandboxName: string, expectedContainerId?: string) { return { run: (cmd: string[], input?: string) => { const result = dockerSpawnSync( - privilegedSandboxExecArgv(sandboxName, cmd, input !== undefined, true), + privilegedSandboxExecArgv(sandboxName, cmd, input !== undefined, true, expectedContainerId), { encoding: "utf-8", input, @@ -2599,7 +2630,10 @@ function repairMutableConfigPerms(sandboxName: string): MutableConfigRepairResul ); } -function restoreLockedStateDirStartupAccess(sandboxName: string): void { +function restoreLockedStateDirStartupAccess( + sandboxName: string, + expectedContainerId?: string, +): void { validateName(sandboxName, "sandbox name"); withExpiredAutoRestoreDeadlineFence( sandboxName, @@ -2609,7 +2643,7 @@ function restoreLockedStateDirStartupAccess(sandboxName: string): void { if (!posture.locked) return; const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); const issues = restoreStateDirStartupAccess( - stateDirLockExec(sandboxName), + stateDirLockExec(sandboxName, expectedContainerId), target.configDir, requireStateLockPlan(target), ); @@ -2617,6 +2651,7 @@ function restoreLockedStateDirStartupAccess(sandboxName: string): void { throw new Error(`Locked startup access could not be restored: ${issues.join(", ")}`); } }, + { allowInlineRecovery: expectedContainerId === undefined }, ); } diff --git a/src/lib/shields/startup-access.test.ts b/src/lib/shields/startup-access.test.ts new file mode 100644 index 00000000000..f733cf8a43e --- /dev/null +++ b/src/lib/shields/startup-access.test.ts @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const { dockerExecFileSync, dockerSpawnSync, run } = vi.hoisted(() => ({ + dockerExecFileSync: vi.fn(() => ""), + dockerSpawnSync: vi.fn(() => ({ status: 0, stdout: "", stderr: "" })), + run: vi.fn(() => ({ status: 0 })), +})); + +vi.mock("../runner", () => ({ + ROOT: "/mock/root", + redact: vi.fn((value) => value), + run, + runCapture: vi.fn(() => "version: 1\nnetwork_policies: {}"), + shellQuote: vi.fn((value) => `'${value}'`), + validateName: vi.fn((value) => value), +})); + +vi.mock("../adapters/docker/exec", () => ({ + dockerExecFileSync, + dockerSpawnSync, +})); + +let testRoot: string; + +function stateDir(): string { + return path.join(testRoot, ".nemoclaw", "state"); +} + +describe("pinned Shields startup access", () => { + beforeEach(() => { + testRoot = fs.mkdtempSync(path.join(os.tmpdir(), "shields-startup-access-")); + vi.stubEnv("HOME", testRoot); + vi.resetModules(); + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + fs.rmSync(testRoot, { force: true, recursive: true }); + }); + + it("refuses expired inline auto-restore before mutating a replacement container (#8662)", async () => { + const sandboxName = "openclaw"; + const snapshotPath = path.join(stateDir(), "policy-snapshot-pinned-startup.yaml"); + const statePath = path.join(stateDir(), `shields-${sandboxName}.json`); + const markerPath = path.join(stateDir(), `shields-timer-${sandboxName}.json`); + fs.mkdirSync(stateDir(), { recursive: true }); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync( + statePath, + JSON.stringify({ + shieldsDown: true, + shieldsDownAt: new Date(Date.now() - 5 * 60 * 1000).toISOString(), + shieldsDownPolicy: "permissive", + shieldsDownReason: "testing pinned startup", + shieldsDownTimeout: 300, + shieldsPolicySnapshotPath: snapshotPath, + updatedAt: new Date().toISOString(), + }), + { mode: 0o600 }, + ); + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: 2_147_483_647, + processToken: "a".repeat(32), + restoreAt: new Date(Date.now() - 1_000).toISOString(), + sandboxName, + snapshotPath, + }), + { mode: 0o600 }, + ); + const stateBefore = fs.readFileSync(statePath, "utf8"); + const { restoreLockedStateDirStartupAccess } = await import("./index"); + + let failure: unknown; + try { + restoreLockedStateDirStartupAccess(sandboxName, "b".repeat(64)); + } catch (error) { + failure = error; + } + + expect(failure).toMatchObject({ + code: "NEMOCLAW_SHIELDS_AUTO_RESTORE_REQUIRED", + message: expect.stringContaining( + "Expired Shields auto-restore must complete before startup access can be repaired", + ), + }); + expect(run).not.toHaveBeenCalled(); + expect(dockerExecFileSync).not.toHaveBeenCalled(); + expect(dockerSpawnSync).not.toHaveBeenCalled(); + expect(fs.readFileSync(statePath, "utf8")).toBe(stateBefore); + expect(fs.existsSync(markerPath)).toBe(true); + }); +}); diff --git a/test/e2e/live/hermes-shields-config.test.ts b/test/e2e/live/hermes-shields-config.test.ts index 4c09ae0016d..7d8e0673304 100644 --- a/test/e2e/live/hermes-shields-config.test.ts +++ b/test/e2e/live/hermes-shields-config.test.ts @@ -19,6 +19,7 @@ import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compati import { REPO_ROOT } from "../fixtures/paths.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { stripAnsi } from "./json-envelope.ts"; +import { expectProtectedStopStartRecovery } from "./shields-restart-recovery-evidence.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-hermes-shields"; const GATEWAY_NAME = process.env.OPENSHELL_GATEWAY ?? "nemoclaw"; @@ -27,6 +28,8 @@ const COMPATIBLE_MODEL = "hermes-shields-e2e-model"; const CONFIG_PATH = "/sandbox/.hermes/config.yaml"; const HERMES_DIR = "/sandbox/.hermes"; const STATE_LOCK_PLAN_PATH = "/usr/local/share/nemoclaw/state-lock-plan.json"; +const RESTART_WORKSPACE_MARKER = "/sandbox/.hermes/workspace/.shields-restart-recovery-marker"; +const RESTART_PAIRING_MARKER = "/sandbox/.hermes/pairing/.shields-restart-recovery-marker"; const COMMAND_TIMEOUT_MS = 120_000; validateSandboxName(SANDBOX_NAME); @@ -103,38 +106,6 @@ async function expectShieldsStatus( expect(resultText(status)).toContain(`Shields: ${expected}`); } -async function expectStopStartRecovery( - host: HostCliClient, - posture: "DOWN" | "UP", - artifactPrefix: string, -): Promise { - const stop = await host.nemoclaw([SANDBOX_NAME, "stop"], { - artifactName: `${artifactPrefix}-stop`, - env: commandEnv(), - redactionValues: [COMPATIBLE_API_KEY], - timeoutMs: 5 * 60_000, - }); - assertExitZero(stop, `stop Hermes with shields ${posture.toLowerCase()}`); - - const start = await host.nemoclaw([SANDBOX_NAME, "start"], { - artifactName: `${artifactPrefix}-start`, - env: commandEnv(), - redactionValues: [COMPATIBLE_API_KEY], - timeoutMs: 5 * 60_000, - }); - assertExitZero(start, `start Hermes with shields ${posture.toLowerCase()}`); - - const status = await host.nemoclaw([SANDBOX_NAME, "status"], { - artifactName: `${artifactPrefix}-status`, - env: commandEnv(), - redactionValues: [COMPATIBLE_API_KEY], - timeoutMs: 5 * 60_000, - }); - assertExitZero(status, `read Hermes status with shields ${posture.toLowerCase()}`); - expect(stripAnsi(resultText(status))).toMatch(/Phase:\s*Ready/i); - await expectShieldsStatus(host, posture, `${artifactPrefix}-shields-status`); -} - async function expectMutablePosture(sandbox: SandboxClient, cycle: number): Promise { const result = await sandboxShell( sandbox, @@ -210,7 +181,7 @@ async function completeShieldsCycle( await expectLockedPosture(sandbox, cycle); } -test("hermes-shields-config: stopped Hermes restores under both Shields postures (#6381, #8112)", { +test("hermes-shields-config: stopped Hermes restores under both Shields postures (#6381, #8112, #8662)", { timeout: HERMES_SHIELDS_CONFIG_TEST_TIMEOUT_MS, meta: { e2ePhases: [ @@ -236,6 +207,8 @@ test("hermes-shields-config: stopped Hermes restores under both Shields postures "shields-up establishes the root-owned locked posture", "start restores a stopped Hermes sandbox while shields are up", "start restores a stopped Hermes sandbox while shields are down", + "protected restart preserves the exact container, registry inference selection, workspace marker, Shields receipt, and configuration seals", + "public start returns with a current PID 1 lease, authenticated managed health, Ready status, and required host-forward health", "each successful shields-down returns only after inference.local serves the configured model", "a second down/up cycle completes without corrupting config state", ], @@ -354,11 +327,49 @@ test("hermes-shields-config: stopped Hermes restores under both Shields postures const configHashBefore = triggerLines.at(-1) ?? ""; expect(configHashBefore).toMatch(/^[0-9a-f]{64}$/); + const restartWorkspaceMarker = await sandboxShell( + sandbox, + `umask 077; mkdir -p ${HERMES_DIR}/workspace ${HERMES_DIR}/pairing; printf '%s\n' 'nemoclaw-8662-hermes-restart-state' > ${RESTART_WORKSPACE_MARKER}; printf '%s\n' 'nemoclaw-8662-hermes-pairing-state' > ${RESTART_PAIRING_MARKER}`, + "seed-restart-workspace-marker", + ); + assertExitZero(restartWorkspaceMarker, "seed Hermes restart workspace marker"); + progress.phase("complete first shields cycle"); await completeShieldsCycle(host, sandbox, 1); progress.phase("restart Hermes with shields up"); - await expectStopStartRecovery(host, "UP", "cycle-1-shields-up-start-recovery"); + await expectProtectedStopStartRecovery({ + agent: "hermes", + artifactPrefix: "cycle-1-shields-up-start-recovery", + artifacts, + configPaths: [ + CONFIG_PATH, + `${HERMES_DIR}/.env`, + `${HERMES_DIR}/.config-hash`, + STATE_LOCK_PLAN_PATH, + RESTART_PAIRING_MARKER, + ], + env: commandEnv(), + host, + posture: "UP", + posturePaths: [ + "/sandbox", + HERMES_DIR, + CONFIG_PATH, + `${HERMES_DIR}/.env`, + `${HERMES_DIR}/.config-hash`, + `${HERMES_DIR}/workspace`, + `${HERMES_DIR}/pairing`, + ], + readShieldsStatus: (artifactName) => runShields(host, ["status"], artifactName), + redactionValues: [COMPATIBLE_API_KEY], + requiredForwards: [ + { path: "/api/status", port: 18789 }, + { path: "/health", port: 8642 }, + ], + sandboxName: SANDBOX_NAME, + workspaceMarkerPath: RESTART_WORKSPACE_MARKER, + }); await expectLockedPosture(sandbox, 1); progress.phase("unlock shields and restart Hermes"); @@ -371,7 +382,38 @@ test("hermes-shields-config: stopped Hermes restores under both Shields postures await expectImmediateInferenceRoute(sandbox, 2); await expectShieldsStatus(host, "DOWN", "cycle-2-status-down"); await expectMutablePosture(sandbox, 2); - await expectStopStartRecovery(host, "DOWN", "cycle-2-shields-down-start-recovery"); + await expectProtectedStopStartRecovery({ + agent: "hermes", + artifactPrefix: "cycle-2-shields-down-start-recovery", + artifacts, + configPaths: [ + CONFIG_PATH, + `${HERMES_DIR}/.env`, + `${HERMES_DIR}/.config-hash`, + STATE_LOCK_PLAN_PATH, + RESTART_PAIRING_MARKER, + ], + env: commandEnv(), + host, + posture: "DOWN", + posturePaths: [ + "/sandbox", + HERMES_DIR, + CONFIG_PATH, + `${HERMES_DIR}/.env`, + `${HERMES_DIR}/.config-hash`, + `${HERMES_DIR}/workspace`, + `${HERMES_DIR}/pairing`, + ], + readShieldsStatus: (artifactName) => runShields(host, ["status"], artifactName), + redactionValues: [COMPATIBLE_API_KEY], + requiredForwards: [ + { path: "/api/status", port: 18789 }, + { path: "/health", port: 8642 }, + ], + sandboxName: SANDBOX_NAME, + workspaceMarkerPath: RESTART_WORKSPACE_MARKER, + }); await expectMutablePosture(sandbox, 2); progress.phase("complete second shields cycle"); @@ -409,6 +451,8 @@ test("hermes-shields-config: stopped Hermes restores under both Shields postures postTransitionInferenceRoute: true, shieldsDownStartRecovery: true, shieldsUpStartRecovery: true, + protectedRestartIdentityAndStatePreservation: true, + protectedRestartManagedRecoveryAndReadiness: true, secondCycle: true, }, }); diff --git a/test/e2e/live/shields-config.test.ts b/test/e2e/live/shields-config.test.ts index fd1dc1867be..efc351ee16b 100644 --- a/test/e2e/live/shields-config.test.ts +++ b/test/e2e/live/shields-config.test.ts @@ -35,12 +35,14 @@ import { resumeSupervisorIfPaused, } from "../fixtures/shields-failed-startup.ts"; import { stripAnsi } from "./json-envelope.ts"; +import { expectProtectedStopStartRecovery } from "./shields-restart-recovery-evidence.ts"; const CONFIG_PATH = "/sandbox/.openclaw/openclaw.json"; const CONFIG_DIR = path.dirname(CONFIG_PATH); const CONFIG_HASH_PATH = `${CONFIG_DIR}/.config-hash`; const CONFIG_GUARD_PATH = "/usr/local/lib/nemoclaw/openclaw-config-guard.py"; const STATE_LOCK_PLAN_PATH = "/usr/local/share/nemoclaw/state-lock-plan.json"; +const RESTART_WORKSPACE_MARKER = "/sandbox/.openclaw/workspace/.shields-restart-recovery-marker"; const STARTUP_MARKER_PATHS = [ "/run/nemoclaw/openclaw-config-ready-v1.capability.json", "/run/nemoclaw/openclaw-config-ready.json", @@ -369,7 +371,7 @@ function readTimerMarker(sandboxName: string): { return JSON.parse(fs.readFileSync(TIMER_FILE(sandboxName), "utf8")); } -test("shields-config: live Shields lifecycle restores stopped OpenClaw under both postures (#8112)", { +test("shields-config: live Shields lifecycle restores stopped OpenClaw under both postures (#8112, #8662)", { timeout: TEST_TIMEOUT_MS, meta: { e2ePhases: [ @@ -398,6 +400,8 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot "fresh mutable-default shields down preserves the mutable config posture", "shields up locks config/workspace and config get redacts secrets", "start restores a stopped OpenClaw sandbox while shields are up", + "protected restart preserves the exact container, registry inference selection, workspace marker, Shields receipt, and configuration seals", + "public start returns with a current PID 1 lease, authenticated managed health, Ready status, and required host-forward health", "empty sealed credentials allow traversal but deny sandbox identity access", "host-root chmod-write-chmod tamper is detected as content drift", "a perms-only .config-hash drift is re-sealed by shields up, not failed closed", @@ -617,6 +621,13 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot expect(layoutProbe.exitCode, resultText(layoutProbe)).toBe(0); expect(resultText(layoutProbe).trim()).toBe(""); + const restartWorkspaceMarker = await sandboxShell( + sandbox, + `umask 077; printf '%s\n' 'nemoclaw-8662-openclaw-restart-state' > ${RESTART_WORKSPACE_MARKER}`, + { artifactName: "phase-2-seed-restart-workspace-marker" }, + ); + expect(restartWorkspaceMarker.exitCode, resultText(restartWorkspaceMarker)).toBe(0); + progress.phase("lock config and workspace and inspect redaction"); const shieldsUp = await runNemoclaw(host, [SANDBOX_NAME, "shields", "up"], { artifactName: "phase-3-shields-up", @@ -686,7 +697,29 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot expect(statusUp.stdout).toContain("Shields: UP"); progress.phase("restart OpenClaw with shields up"); - await expectStopStartRecovery(host, "UP", "phase-5a-shields-up-start-recovery"); + await expectProtectedStopStartRecovery({ + agent: "openclaw", + artifactPrefix: "phase-5a-shields-up-start-recovery", + artifacts, + configPaths: [CONFIG_PATH, CONFIG_HASH_PATH, STATE_LOCK_PLAN_PATH], + env: commandEnv(), + host, + posture: "UP", + posturePaths: [ + "/sandbox", + CONFIG_DIR, + CONFIG_PATH, + CONFIG_HASH_PATH, + `${CONFIG_DIR}/workspace`, + `${CONFIG_DIR}/credentials`, + ], + readShieldsStatus: (artifactName) => + runNemoclaw(host, [SANDBOX_NAME, "shields", "status"], { artifactName }), + redactionValues: [apiKey], + requiredForwards: [{ path: "/health", port: 18789 }], + sandboxName: SANDBOX_NAME, + workspaceMarkerPath: RESTART_WORKSPACE_MARKER, + }); const configAfterLockedRestart = await statPath( sandbox, CONFIG_PATH, @@ -844,7 +877,29 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot expect(statusDown.stdout).toMatch(/Auto-lockdown in:|remaining/i); progress.phase("restart OpenClaw with shields down"); - await expectStopStartRecovery(host, "DOWN", "phase-7a-shields-down-start-recovery"); + await expectProtectedStopStartRecovery({ + agent: "openclaw", + artifactPrefix: "phase-7a-shields-down-start-recovery", + artifacts, + configPaths: [CONFIG_PATH, CONFIG_HASH_PATH, STATE_LOCK_PLAN_PATH], + env: commandEnv(), + host, + posture: "DOWN", + posturePaths: [ + "/sandbox", + CONFIG_DIR, + CONFIG_PATH, + CONFIG_HASH_PATH, + `${CONFIG_DIR}/workspace`, + `${CONFIG_DIR}/credentials`, + ], + readShieldsStatus: (artifactName) => + runNemoclaw(host, [SANDBOX_NAME, "shields", "status"], { artifactName }), + redactionValues: [apiKey], + requiredForwards: [{ path: "/health", port: 18789 }], + sandboxName: SANDBOX_NAME, + workspaceMarkerPath: RESTART_WORKSPACE_MARKER, + }); const configAfterMutableRestart = await statPath( sandbox, CONFIG_PATH, @@ -1100,6 +1155,8 @@ test("shields-config: live Shields lifecycle restores stopped OpenClaw under bot installedFailedStartupLiveChildRefusal: true, installedFailedStartupChildlessUnlock: true, inheritedMutationLockAcceptedByStateGuard: true, + protectedRestartIdentityAndStatePreservation: true, + protectedRestartManagedRecoveryAndReadiness: true, }, }); }); diff --git a/test/e2e/live/shields-restart-recovery-evidence.ts b/test/e2e/live/shields-restart-recovery-evidence.ts new file mode 100644 index 00000000000..35e04b8c26f --- /dev/null +++ b/test/e2e/live/shields-restart-recovery-evidence.ts @@ -0,0 +1,1130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { privilegedSandboxExecArgv } from "../../../src/lib/sandbox/privileged-exec.ts"; +import type { ArtifactSink } from "../fixtures/artifacts.ts"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/command.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { stripAnsi } from "./json-envelope.ts"; + +type RestartAgent = "hermes" | "openclaw"; +type ShieldsPosture = "DOWN" | "UP"; + +type ContainerState = { + readonly created: string; + readonly exitCode: number; + readonly finishedAt: string; + readonly health: string; + readonly id: string; + readonly image: string; + readonly name: string; + readonly pid: number; + readonly restarting: boolean; + readonly running: boolean; + readonly startedAt: string; + readonly status: string; +}; + +type FileEvidence = { + readonly exists: boolean; + readonly gid?: number; + readonly kind?: "directory" | "other" | "regular" | "symlink"; + readonly mode?: string; + readonly nlink?: number; + readonly path: string; + readonly sha256?: string; + readonly uid?: number; +}; + +type ProcessIdentity = { + readonly exists: boolean; + readonly namespaceInode?: number; + readonly parentPid?: number; + readonly pid: number; + readonly startTime?: string; + readonly state?: string; + readonly uids?: readonly number[]; +}; + +type ReadinessMarker = FileEvidence & { + readonly identity?: { + readonly namespaceInode: number; + readonly pid: number; + readonly startTime: string; + readonly version: number; + }; + readonly valid: boolean; +}; + +type ManagedControlEvidence = { + readonly disposition: "already-running"; + readonly exitCode: number; + readonly newPid: number; + readonly nonceSha256: string; + readonly oldPid: number; + readonly phase: "complete"; + readonly valid: true; + readonly version: "v1"; +}; + +type ManagedProcessChainEvidence = { + readonly gateway: ProcessIdentity; + readonly supervisor: ProcessIdentity; +}; + +type RuntimeEvidence = { + readonly files: readonly FileEvidence[]; + readonly pid1: ProcessIdentity; + readonly readiness: readonly ReadinessMarker[]; + readonly runDirectory: FileEvidence; +}; + +type HostStateEvidence = { + readonly configurationSealHash: string; + readonly exists: true; + readonly fileHashes: Record; + readonly gid: number; + readonly mode: string; + readonly policySnapshotHash: string | null; + readonly shieldsDown: boolean; + readonly stateFileHash: string; + readonly uid: number; +}; + +type RegistryEvidence = { + readonly agent: unknown; + readonly entryHash: string; + readonly inferenceHash: string; + readonly model: unknown; + readonly preferredInferenceApi: unknown; + readonly provider: unknown; +}; + +type StageEvidence = { + readonly container: ContainerState; + readonly forwardList: string; + readonly openshellExitCode: number | null; + readonly openshellPhase: string | null; + readonly openshellSandbox: string; + readonly registry: RegistryEvidence; + readonly runtime: RuntimeEvidence | null; + readonly shields: HostStateEvidence; + readonly stage: "after" | "before" | "stopped"; +}; + +export type ShieldsRestartRecoveryOptions = { + readonly agent: RestartAgent; + readonly artifactPrefix: string; + readonly artifacts: ArtifactSink; + readonly configPaths: readonly string[]; + readonly env: NodeJS.ProcessEnv; + readonly host: HostCliClient; + readonly posture: ShieldsPosture; + readonly posturePaths: readonly string[]; + readonly readShieldsStatus: (artifactName: string) => Promise; + readonly redactionValues: readonly string[]; + readonly requiredForwards: readonly { + readonly path: string; + readonly port: number; + }[]; + readonly sandboxName: string; + readonly workspaceMarkerPath: string; +}; + +const RUNTIME_EVIDENCE_SCRIPT = String.raw` +import hashlib +import json +import os +import re +import stat +import sys + +agent = sys.argv[1] +workspace_marker = sys.argv[2] +config_paths = json.loads(sys.argv[3]) +posture_paths = json.loads(sys.argv[4]) +max_file_bytes = 64 * 1024 * 1024 + +def kind(mode): + if stat.S_ISREG(mode): + return "regular" + if stat.S_ISDIR(mode): + return "directory" + if stat.S_ISLNK(mode): + return "symlink" + return "other" + +def metadata(path): + try: + current = os.lstat(path) + except FileNotFoundError: + return {"path": path, "exists": False} + return { + "path": path, + "exists": True, + "kind": kind(current.st_mode), + "mode": format(stat.S_IMODE(current.st_mode), "04o"), + "uid": current.st_uid, + "gid": current.st_gid, + "nlink": current.st_nlink, + } + +def read_regular(path, limit=max_file_bytes): + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1: + raise RuntimeError(f"unsafe regular-file evidence path: {path}") + chunks = [] + total = 0 + while True: + chunk = os.read(descriptor, min(65536, limit + 1 - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > limit: + raise RuntimeError(f"evidence file exceeds limit: {path}") + after = os.fstat(descriptor) + if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ): + raise RuntimeError(f"evidence file changed while reading: {path}") + return b"".join(chunks), before + finally: + os.close(descriptor) + +def file_evidence(path): + record = metadata(path) + if not record["exists"] or record.get("kind") != "regular": + return record + raw, current = read_regular(path) + record.update({ + "mode": format(stat.S_IMODE(current.st_mode), "04o"), + "uid": current.st_uid, + "gid": current.st_gid, + "nlink": current.st_nlink, + "sha256": hashlib.sha256(raw).hexdigest(), + }) + return record + +def proc_identity(pid): + base = f"/proc/{pid}" + try: + raw_stat = open(f"{base}/stat", "r", encoding="ascii").read().strip() + suffix = raw_stat.rsplit(") ", 1)[1].split() + status = open(f"{base}/status", "r", encoding="ascii").read().splitlines() + uid_line = next(line for line in status if line.startswith("Uid:")) + uids = [int(value) for value in uid_line.split()[1:5]] + return { + "pid": pid, + "exists": True, + "parentPid": int(suffix[1]), + "state": suffix[0], + "startTime": suffix[19], + "namespaceInode": os.stat(f"{base}/ns/pid").st_ino, + "uids": uids, + } + except (FileNotFoundError, IndexError, StopIteration, ValueError): + return {"pid": pid, "exists": False} + +pid1 = proc_identity(1) +if not pid1.get("exists"): + raise RuntimeError("PID 1 identity is unavailable") + +def marker(path): + record = file_evidence(path) + record["valid"] = False + if not record.get("exists") or record.get("kind") != "regular": + return record + raw, _current = read_regular(path, 4096) + try: + if agent == "openclaw": + payload = json.loads(raw.decode("ascii")) + identity = { + "version": payload.get("version"), + "pid": payload.get("pid"), + "startTime": str(payload.get("pidStartTime", "")), + "namespaceInode": payload.get("pidNamespaceInode"), + } + else: + match = re.fullmatch(rb"v2 ([0-9]+) ([0-9]+)\n", raw) + if match is None: + return record + identity = { + "version": 2, + "pid": 1, + "startTime": match.group(1).decode("ascii"), + "namespaceInode": int(match.group(2), 10), + } + except (UnicodeDecodeError, ValueError, json.JSONDecodeError): + return record + record["identity"] = identity + record["valid"] = bool( + record.get("mode") == "0600" + and record.get("uid") == 0 + and record.get("gid") == 0 + and record.get("nlink") == 1 + and identity == { + "version": 2, + "pid": 1, + "startTime": pid1["startTime"], + "namespaceInode": pid1["namespaceInode"], + } + ) + return record + +marker_paths = ( + [ + "/run/nemoclaw/openclaw-config-ready-v1.capability.json", + "/run/nemoclaw/openclaw-config-ready.json", + ] + if agent == "openclaw" + else ["/run/nemoclaw/hermes-startup-ready"] +) +all_paths = list(dict.fromkeys(config_paths + posture_paths + [workspace_marker])) +print(json.dumps({ + "pid1": pid1, + "runDirectory": metadata("/run/nemoclaw"), + "readiness": [marker(item) for item in marker_paths], + "files": [file_evidence(item) for item in all_paths], +}, sort_keys=True, separators=(",", ":"))) +`; + +const MANAGED_PROCESS_CHAIN_SCRIPT = String.raw` +import json +import os +import sys + +def proc_identity(pid): + base = f"/proc/{pid}" + try: + raw_stat = open(f"{base}/stat", "r", encoding="ascii").read().strip() + suffix = raw_stat.rsplit(") ", 1)[1].split() + status = open(f"{base}/status", "r", encoding="ascii").read().splitlines() + uid_line = next(line for line in status if line.startswith("Uid:")) + return { + "pid": pid, + "exists": True, + "parentPid": int(suffix[1]), + "state": suffix[0], + "startTime": suffix[19], + "namespaceInode": os.stat(f"{base}/ns/pid").st_ino, + "uids": [int(value) for value in uid_line.split()[1:5]], + } + except (FileNotFoundError, IndexError, StopIteration, ValueError): + return {"pid": pid, "exists": False} + +gateway = proc_identity(int(sys.argv[1], 10)) +supervisor = proc_identity(gateway.get("parentPid", 0)) +print(json.dumps({"gateway": gateway, "supervisor": supervisor}, sort_keys=True, separators=(",", ":"))) +`; + +const RESTART_DIAGNOSTICS_SCRIPT = String.raw` +set +e +sandbox_name="$1" + +redact_stream() { + LC_ALL=C tr -d '\000-\010\013-\037\177' | sed -E \ + -e 's/(nvapi-|sk-)[[:alnum:]_.-]+//g' \ + -e 's/(Bearer[[:space:]]+)[[:alnum:]_.-]+/\1/g' \ + -e 's/(API_SERVER_KEY=)[^[:space:]]+/\1/g' \ + -e 's/(COMPATIBLE_API_KEY=)[^[:space:]]+/\1/g' \ + -e 's/([[:alnum:]_]*(KEY|TOKEN|SECRET|PASSWORD)=)[^[:space:]]+/\1/gI' +} + +printf '%s\n' '== OpenShell sandbox state ==' +openshell sandbox get "$sandbox_name" 2>&1 | redact_stream +printf '%s\n' '== OpenShell phase history checkpoint: forwards ==' +openshell forward list 2>&1 | redact_stream +printf '%s\n' '== OpenShell gateway service ==' +systemctl --user status nemoclaw-openshell-gateway --no-pager -l 2>&1 | redact_stream +printf '%s\n' '== OpenShell gateway journal ==' +journalctl --user -u nemoclaw-openshell-gateway -n 120 --no-pager 2>&1 | redact_stream + +container_ids="$(docker ps -aq --no-trunc \ + --filter label=openshell.ai/managed-by=openshell \ + --filter "label=openshell.ai/sandbox-name=$sandbox_name" \ + --filter label=openshell.ai/sandbox-workspace=default)" +printf '%s\n' '== matching containers ==' +if [ -z "$container_ids" ]; then + printf '%s\n' 'none' +fi +for container_id in $container_ids; do + docker inspect --format '{{.Id}} {{.Name}} {{.Created}} {{.Image}} {{.State.Status}} {{.State.Running}} {{.State.Restarting}} {{.State.Pid}} {{.State.ExitCode}} {{.State.StartedAt}} {{.State.FinishedAt}}' "$container_id" 2>&1 + printf '%s\n' "== container $container_id process tree ==" + docker top "$container_id" -eo pid,ppid,user,stat,comm 2>&1 + printf '%s\n' "== container $container_id runtime logs ==" + docker exec --user 0 "$container_id" sh -lc ' + printf "%s\n" "== PID 1 ==" + ps -o user=,pid=,ppid=,stat=,comm= -p 1 2>&1 || true + printf "%s\n" "== startup readiness leases ==" + stat -c "%n %F %a %u:%g %h" /run/nemoclaw/*ready* 2>&1 || true + printf "%s\n" "== startup log ==" + tail -n 240 /tmp/nemoclaw-start.log 2>&1 || true + printf "%s\n" "== gateway log ==" + tail -n 240 /tmp/gateway.log 2>&1 || true + printf "%s\n" "== Hermes dashboard log ==" + tail -n 120 /tmp/dashboard.log 2>&1 || true + ' 2>&1 | redact_stream + printf '%s\n' "== container $container_id Docker logs ==" + docker logs --tail 240 "$container_id" 2>&1 | redact_stream +done +`; + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function stableJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map((item) => stableJson(item)).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => `${JSON.stringify(key)}:${stableJson(entry)}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + +function fileMode(mode: number): string { + return (mode & 0o7777).toString(8).padStart(4, "0"); +} + +function readStableRegularFile(filePath: string): { raw: Buffer; stats: fs.BigIntStats } { + const descriptor = fs.openSync(filePath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if (!before.isFile() || before.nlink !== 1n || before.size > 64n * 1024n * 1024n) { + throw new Error(`unsafe restart evidence file: ${filePath}`); + } + const raw = fs.readFileSync(descriptor); + const after = fs.fstatSync(descriptor, { bigint: true }); + expect( + [after.dev, after.ino, after.size, after.mtimeNs], + `restart evidence file changed while reading: ${filePath}`, + ).toEqual([before.dev, before.ino, before.size, before.mtimeNs]); + return { raw, stats: after }; + } finally { + fs.closeSync(descriptor); + } +} + +function readRegistryEvidence(sandboxName: string): RegistryEvidence { + const registryPath = path.join(process.env.HOME ?? os.homedir(), ".nemoclaw", "sandboxes.json"); + const registry = JSON.parse(readStableRegularFile(registryPath).raw.toString("utf8")) as { + sandboxes?: Record>; + }; + const entry = registry.sandboxes?.[sandboxName]; + expect(entry, `registry entry ${sandboxName} is missing`).toBeTruthy(); + if (!entry) throw new Error(`registry entry ${sandboxName} is missing`); + const inference = { + compatibleEndpointReasoning: entry.compatibleEndpointReasoning ?? null, + compatibleEndpointReasoningEffort: entry.compatibleEndpointReasoningEffort ?? null, + credentialEnv: entry.credentialEnv ?? null, + endpointSource: entry.endpointSource ?? null, + endpointUrl: entry.endpointUrl ?? null, + model: entry.model ?? null, + nimContainer: entry.nimContainer ?? null, + preferredInferenceApi: entry.preferredInferenceApi ?? null, + provider: entry.provider ?? null, + }; + return { + agent: entry.agent ?? null, + entryHash: sha256(stableJson(entry)), + inferenceHash: sha256(stableJson(inference)), + model: entry.model ?? null, + preferredInferenceApi: entry.preferredInferenceApi ?? null, + provider: entry.provider ?? null, + }; +} + +function readShieldsEvidence(sandboxName: string): HostStateEvidence { + const statePath = path.join( + process.env.HOME ?? os.homedir(), + ".nemoclaw", + "state", + `shields-${sandboxName}.json`, + ); + const { raw, stats: pathMetadata } = readStableRegularFile(statePath); + const state = JSON.parse(raw.toString("utf8")) as { + fileHashes?: Record; + shieldsDown?: boolean; + shieldsDownPolicy?: string | null; + shieldsPolicySnapshotPath?: string | null; + }; + const fileHashes = Object.fromEntries( + Object.entries(state.fileHashes ?? {}).sort(([left], [right]) => left.localeCompare(right)), + ); + return { + configurationSealHash: sha256(stableJson(fileHashes)), + exists: true, + fileHashes, + gid: Number(pathMetadata.gid), + mode: fileMode(Number(pathMetadata.mode)), + policySnapshotHash: state.shieldsPolicySnapshotPath + ? sha256(readStableRegularFile(state.shieldsPolicySnapshotPath).raw) + : null, + shieldsDown: state.shieldsDown === true, + stateFileHash: sha256(raw), + uid: Number(pathMetadata.uid), + }; +} + +function openshellPhase(value: string): string | null { + const plain = stripAnsi(value); + return ( + plain.match(/Phase:\s*([A-Za-z][A-Za-z -]*)/i)?.[1]?.trim() ?? + plain.match(/\b(Ready|Stopped|Provisioning|Running|Failed|Error)\b/i)?.[1] ?? + null + ); +} + +async function containerState( + host: HostCliClient, + sandboxName: string, + artifactPrefix: string, + env: NodeJS.ProcessEnv, + redactionValues: readonly string[], +): Promise { + const list = await host.command( + "docker", + [ + "ps", + "-aq", + "--no-trunc", + "--filter", + "label=openshell.ai/managed-by=openshell", + "--filter", + `label=openshell.ai/sandbox-name=${sandboxName}`, + "--filter", + "label=openshell.ai/sandbox-workspace=default", + ], + { + artifactName: `${artifactPrefix}-docker-containers`, + env, + redactionValues: [...redactionValues], + timeoutMs: 30_000, + }, + ); + expect(list.exitCode, resultText(list)).toBe(0); + const ids = list.stdout.trim().split(/\s+/).filter(Boolean); + expect(ids, `expected exactly one Docker container for ${sandboxName}`).toHaveLength(1); + const inspect = await host.command( + "docker", + [ + "inspect", + "--format", + "{{.Id}}\t{{.Name}}\t{{.Created}}\t{{.Image}}\t{{.State.Status}}\t{{.State.Running}}\t{{.State.Restarting}}\t{{.State.Pid}}\t{{.State.ExitCode}}\t{{.State.StartedAt}}\t{{.State.FinishedAt}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}", + ids[0]!, + ], + { + artifactName: `${artifactPrefix}-docker-inspect`, + env, + redactionValues: [...redactionValues], + timeoutMs: 30_000, + }, + ); + expect(inspect.exitCode, resultText(inspect)).toBe(0); + const fields = inspect.stdout.trim().split("\t"); + expect(fields, `unexpected Docker inspect evidence: ${inspect.stdout}`).toHaveLength(12); + return { + id: fields[0] ?? "", + name: fields[1] ?? "", + created: fields[2] ?? "", + image: fields[3] ?? "", + status: fields[4] ?? "", + running: fields[5] === "true", + restarting: fields[6] === "true", + pid: Number.parseInt(fields[7] ?? "0", 10), + exitCode: Number.parseInt(fields[8] ?? "0", 10), + startedAt: fields[9] ?? "", + finishedAt: fields[10] ?? "", + health: fields[11] ?? "", + }; +} + +async function runtimeEvidence( + host: HostCliClient, + options: ShieldsRestartRecoveryOptions, + containerId: string, + artifactPrefix: string, +): Promise { + const result = await host.command( + "docker", + [ + "exec", + "--user", + "0", + containerId, + "python3", + "-I", + "-c", + RUNTIME_EVIDENCE_SCRIPT, + options.agent, + options.workspaceMarkerPath, + JSON.stringify(options.configPaths), + JSON.stringify(options.posturePaths), + ], + { + artifactName: `${artifactPrefix}-runtime-security-evidence`, + env: options.env, + redactionValues: [...options.redactionValues], + timeoutMs: 60_000, + }, + ); + expect(result.exitCode, resultText(result)).toBe(0); + return JSON.parse(result.stdout.trim()) as RuntimeEvidence; +} + +async function postStartManagedControlProbe( + options: ShieldsRestartRecoveryOptions, + expectedContainerId: string, +): Promise { + const nonce = randomBytes(32).toString("hex"); + const result = await options.host.command( + "docker", + privilegedSandboxExecArgv( + options.sandboxName, + ["/usr/local/bin/nemoclaw-gateway-control", "probe", nonce], + false, + true, + expectedContainerId, + ), + { + artifactName: `${options.artifactPrefix}-post-start-managed-control`, + env: buildAvailabilityProbeEnv(options.env), + // ShellProbe redacts before returning output. The random nonce is a + // single-use challenge, not a credential, and must remain visible long + // enough to authenticate the controller response. Credential values + // still pass through the canonical fixture redaction boundary. + redactionValues: [...options.redactionValues], + timeoutMs: 210_000, + }, + ); + const match = result + ? new RegExp( + `^v1 ${nonce} complete already-running ([1-9][0-9]*) ([1-9][0-9]*)\\nGATEWAY_PID=([1-9][0-9]*)$`, + ).exec(result.stdout.trim()) + : null; + const oldPid = Number.parseInt(match?.[1] ?? "0", 10); + const newPid = Number.parseInt(match?.[2] ?? "0", 10); + const gatewayPid = Number.parseInt(match?.[3] ?? "0", 10); + if ( + result.exitCode !== 0 || + result.stderr.trim() !== "" || + match === null || + !Number.isSafeInteger(oldPid) || + !Number.isSafeInteger(newPid) || + oldPid !== newPid || + newPid !== gatewayPid + ) { + throw new Error( + `post-start managed-control authentication failed with exit code ${result.exitCode}; ` + + `inspect the redacted ${options.artifactPrefix}-post-start-managed-control artifact`, + ); + } + const evidence: ManagedControlEvidence = { + disposition: "already-running", + exitCode: 0, + newPid, + nonceSha256: sha256(nonce), + oldPid, + phase: "complete", + valid: true, + version: "v1", + }; + await options.artifacts.writeJson( + `${options.artifactPrefix}-post-start-managed-control.json`, + evidence, + ); + return evidence; +} + +async function managedProcessChainEvidence( + options: ShieldsRestartRecoveryOptions, + containerId: string, + gatewayPid: number, +): Promise { + const result = await options.host.command( + "docker", + [ + "exec", + "--user", + "0", + containerId, + "python3", + "-I", + "-c", + MANAGED_PROCESS_CHAIN_SCRIPT, + String(gatewayPid), + ], + { + artifactName: `${options.artifactPrefix}-post-start-managed-process-chain`, + env: options.env, + redactionValues: [...options.redactionValues], + timeoutMs: 30_000, + }, + ); + expect(result.exitCode, resultText(result)).toBe(0); + return JSON.parse(result.stdout.trim()) as ManagedProcessChainEvidence; +} + +async function captureStage( + options: ShieldsRestartRecoveryOptions, + stage: StageEvidence["stage"], + includeRuntime: boolean, +): Promise { + const prefix = `${options.artifactPrefix}-${stage}`; + const sandboxState = await options.host.command( + "openshell", + ["sandbox", "get", options.sandboxName], + { + artifactName: `${prefix}-openshell-sandbox`, + env: options.env, + redactionValues: [...options.redactionValues], + timeoutMs: 60_000, + }, + ); + if (stage !== "stopped") { + expect(sandboxState.exitCode, resultText(sandboxState)).toBe(0); + } + const forwards = await options.host.command("openshell", ["forward", "list"], { + artifactName: `${prefix}-openshell-forwards`, + env: options.env, + redactionValues: [...options.redactionValues], + timeoutMs: 60_000, + }); + expect(forwards.exitCode, resultText(forwards)).toBe(0); + const container = await containerState( + options.host, + options.sandboxName, + prefix, + options.env, + options.redactionValues, + ); + const evidence: StageEvidence = { + container, + forwardList: stripAnsi(forwards.stdout), + openshellExitCode: sandboxState.exitCode, + openshellPhase: openshellPhase(resultText(sandboxState)), + openshellSandbox: stripAnsi(resultText(sandboxState)), + registry: readRegistryEvidence(options.sandboxName), + runtime: includeRuntime + ? await runtimeEvidence(options.host, options, container.id, prefix) + : null, + shields: readShieldsEvidence(options.sandboxName), + stage, + }; + await options.artifacts.writeJson(`${prefix}-restart-evidence.json`, evidence); + return evidence; +} + +async function captureDiagnostics( + options: ShieldsRestartRecoveryOptions, + stage: string, +): Promise { + await options.host.command( + "bash", + [ + "-lc", + RESTART_DIAGNOSTICS_SCRIPT, + "shields-restart-recovery-diagnostics", + options.sandboxName, + ], + { + artifactName: `${options.artifactPrefix}-${stage}-diagnostics`, + captureLimitBytes: 512 * 1024, + env: options.env, + redactionValues: [...options.redactionValues], + timeoutMs: 90_000, + }, + ); +} + +function assertImmutableContainer( + before: ContainerState, + current: ContainerState, + stage: string, +): void { + expect(current.id, `${stage} replaced the Docker container`).toBe(before.id); + expect(current.name, `${stage} renamed the Docker container`).toBe(before.name); + expect(current.created, `${stage} changed Docker creation identity`).toBe(before.created); + expect(current.image, `${stage} changed the Docker image identity`).toBe(before.image); +} + +function assertRuntimeReady( + runtime: RuntimeEvidence, + agent: RestartAgent, + requiredFilePaths: ReadonlySet, + requiredDirectoryPaths: ReadonlySet, +): void { + expect(runtime.runDirectory).toMatchObject({ exists: true, gid: 0, kind: "directory", uid: 0 }); + expect(runtime.runDirectory.mode).toMatch( + agent === "openclaw" ? /^07(?:00|11)$/ : /^07(?:11|55)$/, + ); + expect(runtime.pid1).toMatchObject({ exists: true, parentPid: 0, pid: 1 }); + expect(runtime.pid1.state).not.toBe("Z"); + expect(runtime.pid1.uids).toEqual([0, 0, 0, 0]); + expect(runtime.readiness).toHaveLength(agent === "openclaw" ? 2 : 1); + for (const marker of runtime.readiness) { + expect(marker.valid, `invalid startup readiness lease at ${marker.path}`).toBe(true); + } + if (agent === "openclaw") { + expect(runtime.readiness[0]?.identity).toEqual(runtime.readiness[1]?.identity); + } + const files = new Map(runtime.files.map((entry) => [entry.path, entry])); + for (const filePath of requiredFilePaths) { + expect(files.get(filePath), `required restart file is missing: ${filePath}`).toMatchObject({ + exists: true, + kind: "regular", + nlink: 1, + sha256: expect.stringMatching(/^[0-9a-f]{64}$/), + }); + } + for (const directoryPath of requiredDirectoryPaths) { + expect( + files.get(directoryPath), + `required restart directory is missing: ${directoryPath}`, + ).toMatchObject({ exists: true, kind: "directory" }); + } +} + +function assertRestartFileInvariants( + before: RuntimeEvidence, + after: RuntimeEvidence, + options: ShieldsRestartRecoveryOptions, +): void { + const beforeFiles = new Map(before.files.map((entry) => [entry.path, entry])); + const afterFiles = new Map(after.files.map((entry) => [entry.path, entry])); + const sandboxGid = afterFiles.get("/sandbox")?.gid; + expect([...afterFiles.keys()]).toEqual([...beforeFiles.keys()]); + const openClawCredentials = "/sandbox/.openclaw/credentials"; + for (const [filePath, beforeFile] of beforeFiles) { + const afterFile = afterFiles.get(filePath); + expect(afterFile, `restart evidence disappeared for ${filePath}`).toBeDefined(); + if ( + options.agent === "openclaw" && + options.posture === "UP" && + filePath === openClawCredentials + ) { + expect(beforeFile).toMatchObject({ exists: true, kind: "directory", uid: 0 }); + expect(beforeFile.mode).toMatch(/^07(?:00|10)$/); + expect(afterFile).toMatchObject({ exists: true, kind: "directory", mode: "0710", uid: 0 }); + expect(sandboxGid, "sandbox group identity is unavailable").toBeGreaterThan(0); + expect(afterFile?.gid, "OpenClaw credentials traversal must use the sandbox group").toBe( + sandboxGid, + ); + continue; + } + if (beforeFile.kind === "directory") { + const { nlink: _beforeNlink, ...stableBefore } = beforeFile; + const { nlink: _afterNlink, ...stableAfter } = afterFile!; + expect(stableAfter, `restart changed protected directory state at ${filePath}`).toEqual( + stableBefore, + ); + } else { + expect(afterFile, `restart changed protected state at ${filePath}`).toEqual(beforeFile); + } + } +} + +function assertManagedControlReady( + control: ManagedControlEvidence, + processChain: ManagedProcessChainEvidence, + pid1: ProcessIdentity, +): void { + expect(control.valid, "the post-start managed-control probe was not authenticated").toBe(true); + expect(control).toMatchObject({ + disposition: "already-running", + exitCode: 0, + phase: "complete", + version: "v1", + }); + expect(control.nonceSha256).toMatch(/^[0-9a-f]{64}$/); + expect(control.newPid).toBe(control.oldPid); + expect(processChain.gateway).toMatchObject({ exists: true, pid: control.newPid }); + expect(processChain.gateway.parentPid).toBeGreaterThan(1); + expect(processChain.gateway.state).not.toBe("Z"); + expect(processChain.gateway.namespaceInode).toBe(pid1.namespaceInode); + expect(processChain.supervisor).toMatchObject({ + exists: true, + parentPid: 1, + pid: processChain.gateway.parentPid, + }); + expect(processChain.supervisor.state).not.toBe("Z"); + expect(processChain.supervisor.namespaceInode).toBe(pid1.namespaceInode); + const gatewayUids = processChain.gateway.uids ?? []; + expect(gatewayUids).toHaveLength(4); + expect(new Set(gatewayUids).size).toBe(1); + expect(gatewayUids[0], "managed gateway must run as a non-root identity").toBeGreaterThan(0); + expect(processChain.supervisor.uids).toEqual(gatewayUids); +} + +function assertRequiredForward(forwardList: string, sandboxName: string, port: number): void { + const rows = forwardList + .split(/\r?\n/) + .map((line) => line.trim().split(/\s+/)) + .filter( + (columns) => + columns[0] === sandboxName && + columns[2] === String(port) && + /^(active|running)$/i.test(columns[4] ?? ""), + ); + expect( + rows, + `required host forward ${sandboxName}:${port} must have one active owner`, + ).toHaveLength(1); + const row = rows[0]!; + expect(row[1], `required host forward ${sandboxName}:${port} must bind loopback`).toMatch( + /^(?:127\.0\.0\.1|\[?::1\]?)$/, + ); + expect(Number.parseInt(row[3] ?? "0", 10)).toBeGreaterThan(0); +} + +async function assertForwardHealth( + options: ShieldsRestartRecoveryOptions, + port: number, + healthPath: string, +): Promise { + const health = await options.host.command( + "curl", + [ + "-q", + "--noproxy", + "*", + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--connect-timeout", + "3", + "--max-time", + "10", + `http://127.0.0.1:${port}${healthPath}`, + ], + { + artifactName: `${options.artifactPrefix}-after-forward-${port}-health`, + env: options.env, + redactionValues: [...options.redactionValues], + timeoutMs: 30_000, + }, + ); + expect(health.exitCode, resultText(health)).toBe(0); + expect(health.stdout.trim(), resultText(health)).toMatch(/^(200|401)$/); +} + +async function captureStartReturnBarrier(options: ShieldsRestartRecoveryOptions): Promise<{ + forwardList: string; + openshellPhase: string | null; +}> { + const sandboxState = await options.host.command( + "openshell", + ["sandbox", "get", options.sandboxName], + { + artifactName: `${options.artifactPrefix}-return-barrier-openshell-sandbox`, + env: options.env, + redactionValues: [...options.redactionValues], + timeoutMs: 60_000, + }, + ); + expect(sandboxState.exitCode, resultText(sandboxState)).toBe(0); + const phase = openshellPhase(resultText(sandboxState)); + expect(phase, "public start returned before OpenShell reported Ready").toMatch(/^Ready$/i); + + const forwards = await options.host.command("openshell", ["forward", "list"], { + artifactName: `${options.artifactPrefix}-return-barrier-openshell-forwards`, + env: options.env, + redactionValues: [...options.redactionValues], + timeoutMs: 60_000, + }); + expect(forwards.exitCode, resultText(forwards)).toBe(0); + const forwardList = stripAnsi(forwards.stdout); + for (const required of options.requiredForwards) { + assertRequiredForward(forwardList, options.sandboxName, required.port); + await assertForwardHealth(options, required.port, required.path); + } + return { forwardList, openshellPhase: phase }; +} + +function leaseIdentity(runtime: RuntimeEvidence): string { + return stableJson(runtime.readiness.map((marker) => marker.identity)); +} + +/** + * Exercise the public stop/start path and bind success to the preserved + * container, Shields receipt, config seals, PID 1 lease, a nonce-bound + * post-start managed-control authentication probe, and required host forwards. + */ +export async function expectProtectedStopStartRecovery( + options: ShieldsRestartRecoveryOptions, +): Promise { + const requiredFilePaths = new Set([...options.configPaths, options.workspaceMarkerPath]); + const requiredDirectoryPaths = new Set( + options.posturePaths.filter((filePath) => !requiredFilePaths.has(filePath)), + ); + const before = await captureStage(options, "before", true); + expect(before.container.running, "sandbox container must run before stop").toBe(true); + expect(before.container.restarting, "sandbox container must not restart before stop").toBe(false); + expect(before.openshellPhase).toMatch(/^Ready$/i); + expect(before.shields.shieldsDown).toBe(options.posture === "DOWN"); + expect(before.shields.fileHashes).not.toEqual({}); + expect(before.shields.mode).toBe("0600"); + for (const [sealedPath, sealedHash] of Object.entries(before.shields.fileHashes)) { + expect(sealedPath).toMatch(/^\/sandbox\//); + expect(sealedHash).toMatch(/^[0-9a-f]{64}$/); + } + expect(before.registry.agent).toBe(options.agent); + assertRuntimeReady(before.runtime!, options.agent, requiredFilePaths, requiredDirectoryPaths); + for (const required of options.requiredForwards) { + assertRequiredForward(before.forwardList, options.sandboxName, required.port); + } + await captureDiagnostics(options, "before-stop"); + + const stop = await options.host.nemoclaw([options.sandboxName, "stop"], { + artifactName: `${options.artifactPrefix}-stop`, + env: options.env, + redactionValues: [...options.redactionValues], + timeoutMs: 5 * 60_000, + }); + if (stop.exitCode !== 0) { + await captureDiagnostics(options, "stop-failed"); + } + expect(stop.exitCode, resultText(stop)).toBe(0); + + const stopped = await captureStage(options, "stopped", false); + assertImmutableContainer(before.container, stopped.container, "stop"); + expect(stopped.container.running, "public stop left the container running").toBe(false); + expect(stopped.container.restarting, "public stop left the container restarting").toBe(false); + expect(stopped.registry).toEqual(before.registry); + expect(stopped.shields).toEqual(before.shields); + await captureDiagnostics(options, "after-stop-before-start"); + + const start = await options.host.nemoclaw([options.sandboxName, "start"], { + artifactName: `${options.artifactPrefix}-start`, + env: options.env, + redactionValues: [...options.redactionValues], + timeoutMs: 5 * 60_000, + }); + if (start.exitCode !== 0) { + await captureDiagnostics(options, "start-failed"); + } + expect(start.exitCode, resultText(start)).toBe(0); + + let returnBarrier: Awaited>; + let after: StageEvidence; + try { + returnBarrier = await captureStartReturnBarrier(options); + after = await captureStage(options, "after", true); + } catch (error) { + await captureDiagnostics(options, "after-start-evidence-failed"); + throw error; + } + + const status = await options.host.nemoclaw([options.sandboxName, "status"], { + artifactName: `${options.artifactPrefix}-status`, + env: options.env, + redactionValues: [...options.redactionValues], + timeoutMs: 5 * 60_000, + }); + expect(status.exitCode, resultText(status)).toBe(0); + expect(stripAnsi(resultText(status))).toMatch(/Phase:\s*Ready/i); + const shields = await options.readShieldsStatus(`${options.artifactPrefix}-shields-status`); + expect(shields.exitCode, resultText(shields)).toBe(0); + expect(resultText(shields)).toContain(`Shields: ${options.posture}`); + + assertImmutableContainer(before.container, after.container, "start"); + expect(after.container.running, "public start did not leave the container running").toBe(true); + expect(after.container.restarting, "public start left the container restarting").toBe(false); + expect(after.container.pid).toBeGreaterThan(1); + expect(after.container.startedAt).not.toBe(before.container.startedAt); + expect(after.openshellPhase).toMatch(/^Ready$/i); + expect(after.registry).toEqual(before.registry); + expect(after.shields).toEqual(before.shields); + assertRestartFileInvariants(before.runtime!, after.runtime!, options); + assertRuntimeReady(after.runtime!, options.agent, requiredFilePaths, requiredDirectoryPaths); + expect( + after.runtime!.runDirectory.mode, + "public start returned without reaching the managed-control runtime boundary", + ).toBe("0711"); + expect(leaseIdentity(after.runtime!), "restart reused the prior PID 1 readiness lease").not.toBe( + leaseIdentity(before.runtime!), + ); + + for (const required of options.requiredForwards) { + assertRequiredForward(after.forwardList, options.sandboxName, required.port); + } + + let managedControl: ManagedControlEvidence; + let processChain: ManagedProcessChainEvidence; + try { + managedControl = await postStartManagedControlProbe(options, before.container.id); + processChain = await managedProcessChainEvidence( + options, + after.container.id, + managedControl.newPid, + ); + assertManagedControlReady(managedControl, processChain, after.runtime!.pid1); + } catch (error) { + await captureDiagnostics(options, "post-start-managed-control-failed"); + throw error; + } + await captureDiagnostics(options, "after-start"); + + await options.artifacts.writeJson(`${options.artifactPrefix}-restart-recovery-summary.json`, { + agent: options.agent, + configurationSealHash: after.shields.configurationSealHash, + containerId: after.container.id, + publicStartExitCode: start.exitCode, + postStartManagedControlProbe: { + disposition: managedControl.disposition, + newPid: managedControl.newPid, + nonceSha256: managedControl.nonceSha256, + oldPid: managedControl.oldPid, + }, + openshellPhaseHistory: [ + { + exitCode: before.openshellExitCode, + phase: before.openshellPhase, + stage: before.stage, + }, + { + exitCode: stopped.openshellExitCode, + phase: stopped.openshellPhase, + stage: stopped.stage, + }, + { + exitCode: after.openshellExitCode, + phase: after.openshellPhase, + stage: after.stage, + }, + ], + posture: options.posture, + readinessLeaseAfterHash: sha256(leaseIdentity(after.runtime!)), + readinessLeaseBeforeHash: sha256(leaseIdentity(before.runtime!)), + returnBarrier: { + forwardListHash: sha256(returnBarrier.forwardList), + openshellPhase: returnBarrier.openshellPhase, + }, + registryEntryHash: after.registry.entryHash, + registryInferenceHash: after.registry.inferenceHash, + requiredForwards: options.requiredForwards.map(({ port }) => port), + shieldsStateHash: after.shields.stateFileHash, + workspaceMarkerHash: after.runtime!.files.find( + (entry) => entry.path === options.workspaceMarkerPath, + )?.sha256, + }); +} diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index 7bad729a80a..a6b527c71c9 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -733,7 +733,12 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { recovered: true, forwardRecovered: true, }); - expect(requestPinnedGatewaySupervisorAction).toHaveBeenCalledTimes(5); + expect(requestPinnedGatewaySupervisorAction).toHaveBeenCalledTimes(6); + expect( + requestPinnedGatewaySupervisorAction.mock.calls.every( + (call: unknown[]) => call[1] === "probe" && call[3] === "replacement-container-id", + ), + ).toBe(true); expect(captureOpenshell).toHaveBeenCalledWith( ["sandbox", "exec", "--name", "busy-recovered-box", "--", "true"], expect.objectContaining({ ignoreError: true }), From c73beb92bc3bac32f493bb69f098e911dc8ee55a Mon Sep 17 00:00:00 2001 From: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:41:52 +0530 Subject: [PATCH 5/7] fix(sandbox): align managed recovery authority Signed-off-by: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> --- src/lib/actions/sandbox/connect.ts | 6 +- src/lib/actions/sandbox/process-recovery.ts | 19 ++++-- src/lib/actions/sandbox/start.test.ts | 58 +++++++++++++++++++ src/lib/actions/sandbox/start.ts | 13 +++-- ...ocess-recovery-supervisor-relaunch.test.ts | 6 +- 5 files changed, 84 insertions(+), 18 deletions(-) diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 746681c2b90..77022371f70 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -268,11 +268,11 @@ function exitOnPreservedStartupRecoveryFailure( } export function sanitizeSandboxStartupRecoveryDetail(raw: string): string { - return redactFull(raw) + const normalized = raw .replace(/[\u0000-\u001f\u007f-\u009f]/gu, " ") .replace(/\s+/gu, " ") - .trim() - .slice(0, 240); + .trim(); + return redactFull(normalized).slice(0, 240); } async function runSandboxConnectProbe( diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 4e50cbc55ce..7b470018bd3 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -676,16 +676,24 @@ function runningSandboxRecoveryPreflight( }); } -function shouldUseManagedStartupRecovery( - sandboxName: string, +export function shouldUseManagedStartupRecovery( + persistedAgent: string | null | undefined, agent: ReturnType, preserveContainer: boolean, ): boolean { if (!preserveContainer) return false; if (agent) return agent.name === "openclaw" || agent.name === "hermes"; + return !persistedAgent || persistedAgent === "openclaw" || persistedAgent === "hermes"; +} + +function shouldUseManagedStartupRecoveryForSandbox( + sandboxName: string, + agent: ReturnType, + preserveContainer: boolean, +): boolean { try { const persistedAgent = registry.getSandbox(sandboxName)?.agent; - return !persistedAgent || persistedAgent === "openclaw" || persistedAgent === "hermes"; + return shouldUseManagedStartupRecovery(persistedAgent, agent, preserveContainer); } catch { return false; } @@ -705,7 +713,8 @@ function inspectManagedStartupRecovery( } { const running = inspectGateway(sandboxName); const managedStartupRecovery = - running === null && shouldUseManagedStartupRecovery(sandboxName, agent, preserveContainer); + running === null && + shouldUseManagedStartupRecoveryForSandbox(sandboxName, agent, preserveContainer); return { failure: managedStartupRecovery ? managedSupervisorProbeFailure( @@ -1465,7 +1474,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( ? (name, action, timeout = 210000) => requestPinnedGatewaySupervisorAction(name, action, timeout, expectedContainerId) : requestGatewaySupervisorAction; - const requirePreservedManagedHealth = shouldUseManagedStartupRecovery( + const requirePreservedManagedHealth = shouldUseManagedStartupRecoveryForSandbox( sandboxName, recoveryAgent, preserveContainer, diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index 0804122f946..e14e2ce3155 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -121,6 +121,7 @@ function harness(overrides: Partial = {}) { describe("startSandbox", () => { beforeEach(() => { + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null); vi.spyOn(sandboxStartDependencies, "loadConnect").mockReturnValue(connect); vi.spyOn(sandboxStartDependencies, "loadShields").mockReturnValue(shields); }); @@ -149,6 +150,34 @@ describe("startSandbox", () => { expect(result).toBe(recovery); }); + it("pins sealed startup access to the lifecycle container identity (#8662)", () => { + const restoreAccess = vi.fn(); + const recovery = runningStartupState(); + const restoreProcesses = vi.fn(() => recovery); + const processRecoveryOptions = { expectedContainerId: ORIGINAL_CONTAINER_ID }; + + const result = restoreStoppedSandboxStartupState("my-sandbox", { + agent: "openclaw", + processRecovery: processRecoveryOptions, + restoreLockedStartupAccess: restoreAccess, + restoreProcessState: restoreProcesses, + }); + + expect(restoreAccess).toHaveBeenCalledExactlyOnceWith("my-sandbox", ORIGINAL_CONTAINER_ID); + expect(restoreProcesses).toHaveBeenCalledExactlyOnceWith("my-sandbox", processRecoveryOptions); + expect(result).toBe(recovery); + }); + + it("normalizes control characters before redacting startup diagnostics (#8662)", () => { + const sanitized = connect.sanitizeSandboxStartupRecoveryDetail( + "supervisor failed: Authorization:\u0000Bearer opaque-control-token", + ); + + expect(sanitized).toContain("Authorization: Bearer "); + expect(sanitized).not.toContain("opaque-control-token"); + expect(sanitized).not.toMatch(/[\u0000-\u001f\u007f-\u009f]/u); + }); + it("keeps Hermes sealed state untouched while recovering sandbox processes (#8112)", () => { const restoreAccess = vi.fn(); const recovery = runningStartupState(); @@ -434,6 +463,35 @@ describe("startSandbox", () => { expect(h.verifyGateway).toHaveBeenCalledWith("my-sandbox"); }); + it("uses the session agent as the managed-recovery authority when registry state is stale (#8662)", async () => { + const entry = sandbox({ agent: "openclaw", openshellDriver: "docker" }); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ + name: "custom-gateway", + displayName: "Custom Gateway", + forwardPort: 19090, + healthProbe: { + port: 19090, + timeout_seconds: 30, + url: "http://127.0.0.1:19090/health", + }, + runtime: { kind: "gateway" }, + } as never); + vi.spyOn(persistedRegistry, "getSandbox").mockReturnValue(entry); + const h = harness(); + h.getSandbox.mockReturnValue(entry); + h.restoreStartupState.mockReturnValue({ + checked: false, + wasRunning: null, + recovered: false, + forwardRecovered: false, + }); + + await expect(startSandbox("my-sandbox", h.deps)).resolves.toEqual({ exitCode: 0 }); + + expect(h.restoreStartupState).toHaveBeenCalledExactlyOnceWith("my-sandbox"); + expect(h.verifyGateway).toHaveBeenCalledExactlyOnceWith("my-sandbox"); + }); + it.each( (["openclaw", "hermes"] as const).flatMap((agent) => [ { diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 56519c12a17..8bcc1c84762 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import * as agentRuntime from "../../agent/runtime"; import { cliName } from "../../onboard/branding"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, @@ -9,6 +10,7 @@ import { import { SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED } from "../../shields"; import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; +import { shouldUseManagedStartupRecovery } from "./process-recovery"; import { resolveSandboxLifecycleProvider, type SandboxLifecycleResult, @@ -196,10 +198,6 @@ export interface SandboxStartupStateDeps { ) => SandboxProcessRecoveryResult; } -function requiresManagedStartupRecovery(agent: SandboxEntry["agent"]): boolean { - return !agent || agent === "openclaw" || agent === "hermes"; -} - export function restoreStoppedSandboxStartupState( sandboxName: string, deps: SandboxStartupStateDeps = {}, @@ -263,8 +261,11 @@ export async function startSandbox( const result = resolved.lifecycle.start(input); if (result.exitCode !== 0) return result; const expectedContainerId = result.runtimeHandle; - const pinnedManagedStartupRecovery = - requiresManagedStartupRecovery(resolved.sandbox.agent) && Boolean(expectedContainerId); + const pinnedManagedStartupRecovery = shouldUseManagedStartupRecovery( + resolved.sandbox.agent, + agentRuntime.getSessionAgent(sandboxName), + Boolean(expectedContainerId), + ); await resolved.lifecycle.verifyStarted(input, async (name) => { log(" Restoring sandbox startup state…"); diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index a6b527c71c9..7241668d9ac 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -735,10 +735,8 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { }); expect(requestPinnedGatewaySupervisorAction).toHaveBeenCalledTimes(6); expect( - requestPinnedGatewaySupervisorAction.mock.calls.every( - (call: unknown[]) => call[1] === "probe" && call[3] === "replacement-container-id", - ), - ).toBe(true); + requestPinnedGatewaySupervisorAction.mock.calls.map((call: unknown[]) => [call[1], call[3]]), + ).toEqual(Array.from({ length: 6 }, () => ["probe", "replacement-container-id"])); expect(captureOpenshell).toHaveBeenCalledWith( ["sandbox", "exec", "--name", "busy-recovered-box", "--", "true"], expect.objectContaining({ ignoreError: true }), From c0a73ded2f92e11f600ee3300a1409edf8b49504 Mon Sep 17 00:00:00 2001 From: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:50:23 +0530 Subject: [PATCH 6/7] fix(sandbox): require managed runtime identity Signed-off-by: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> --- src/lib/actions/sandbox/start.test.ts | 36 +++++++++++++++++++++++++++ src/lib/actions/sandbox/start.ts | 17 +++++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index e14e2ce3155..d6f19e57ba7 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -492,6 +492,42 @@ describe("startSandbox", () => { expect(h.verifyGateway).toHaveBeenCalledExactlyOnceWith("my-sandbox"); }); + it("fails closed before verification when managed startup returns no runtime identity (#8662)", async () => { + const h = harness(); + const docker = h.deps.runtimeProviders?.docker; + if (!docker || docker.lifecycle.supported !== true) { + throw new Error("test requires the Docker lifecycle provider"); + } + const verifyStarted = vi.fn(); + const runtimeProviders = createRuntimeProviderBundleRegistry([ + [ + "docker", + { + ...docker, + lifecycle: { + ...docker.lifecycle, + start: () => ({ exitCode: 0 }), + verifyStarted, + }, + }, + ], + ]); + h.getSandbox.mockReturnValue(sandbox({ agent: "openclaw", openshellDriver: "docker" })); + + const result = await startSandbox("my-sandbox", { ...h.deps, runtimeProviders }); + + expect(result).toEqual({ + exitCode: 1, + message: + " Sandbox 'my-sandbox' started, but runtime provider 'docker' returned no immutable " + + "runtime identity. Refusing unpinned managed startup recovery; the existing sandbox was " + + "preserved. Run 'nemoclaw doctor', then retry 'nemoclaw my-sandbox start'.", + }); + expect(verifyStarted).not.toHaveBeenCalled(); + expect(h.restoreStartupState).not.toHaveBeenCalled(); + expect(h.verifyGateway).not.toHaveBeenCalled(); + }); + it.each( (["openclaw", "hermes"] as const).flatMap((agent) => [ { diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 8bcc1c84762..6f35bf1fd00 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -261,11 +261,24 @@ export async function startSandbox( const result = resolved.lifecycle.start(input); if (result.exitCode !== 0) return result; const expectedContainerId = result.runtimeHandle; - const pinnedManagedStartupRecovery = shouldUseManagedStartupRecovery( + const managedStartupRecovery = shouldUseManagedStartupRecovery( resolved.sandbox.agent, agentRuntime.getSessionAgent(sandboxName), - Boolean(expectedContainerId), + true, ); + const requiresManagedRuntimeHandle = + managedStartupRecovery && resolved.bundle.identity.id === "docker"; + if (requiresManagedRuntimeHandle && !expectedContainerId) { + return { + exitCode: 1, + message: + ` Sandbox '${sandboxName}' started, but runtime provider '${resolved.bundle.identity.id}' ` + + "returned no immutable runtime identity. Refusing unpinned managed startup recovery; " + + `the existing sandbox was preserved. Run '${cliName()} doctor', then retry ` + + `'${cliName()} ${sandboxName} start'.`, + }; + } + const pinnedManagedStartupRecovery = managedStartupRecovery && Boolean(expectedContainerId); await resolved.lifecycle.verifyStarted(input, async (name) => { log(" Restoring sandbox startup state…"); From 13d6da02d617748ed2302fd723511cfe6c9fc7dd Mon Sep 17 00:00:00 2001 From: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:55:11 +0530 Subject: [PATCH 7/7] test(sandbox): keep identity guard linear Signed-off-by: souvikDevloper <138186578+souvikDevloper@users.noreply.github.com> --- src/lib/actions/sandbox/start.test.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index d6f19e57ba7..dbe5a7f4dc5 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -495,17 +495,20 @@ describe("startSandbox", () => { it("fails closed before verification when managed startup returns no runtime identity (#8662)", async () => { const h = harness(); const docker = h.deps.runtimeProviders?.docker; - if (!docker || docker.lifecycle.supported !== true) { - throw new Error("test requires the Docker lifecycle provider"); - } + expect(docker?.lifecycle.supported).toBe(true); + const dockerProvider = docker as NonNullable; + const dockerLifecycle = dockerProvider.lifecycle as Extract< + NonNullable["lifecycle"], + { readonly supported: true } + >; const verifyStarted = vi.fn(); const runtimeProviders = createRuntimeProviderBundleRegistry([ [ "docker", { - ...docker, + ...dockerProvider, lifecycle: { - ...docker.lifecycle, + ...dockerLifecycle, start: () => ({ exitCode: 0 }), verifyStarted, },