diff --git a/.github/workflows/platform-vitest-main.yaml b/.github/workflows/platform-vitest-main.yaml index 7a00d21c510..9860e4c78c7 100644 --- a/.github/workflows/platform-vitest-main.yaml +++ b/.github/workflows/platform-vitest-main.yaml @@ -326,7 +326,7 @@ jobs: "@ Invoke-WslScript -Distro $env:WSL_DISTRO -User $env:WSL_TEST_USER -Script $script - - name: Run full Vitest suite in WSL + - name: Run Vitest suite in WSL shell: powershell run: | . "$env:TRUSTED_WSL_HELPER" @@ -338,7 +338,8 @@ jobs: cd $workdir export NEMOCLAW_EXEC_TIMEOUT=60000 export NEMOCLAW_TEST_TIMEOUT=60000 - npx vitest run --testTimeout 60000 --shard='${{ matrix.shard }}/4' + npx vitest run --testTimeout 60000 --shard='${{ matrix.shard }}/4' \ + --exclude test/runtime/sandbox/jetson-device-group-bootstrap.test.ts "@ Invoke-WslScript -Distro $env:WSL_DISTRO -User $env:WSL_TEST_USER -Script $script diff --git a/Dockerfile b/Dockerfile index e9b8ca384e8..7b83f7651be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -577,6 +577,7 @@ COPY --chmod=0444 scripts/lib/corporate-ca-runtime.sh /usr/local/lib/nemoclaw/co COPY scripts/lib/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh COPY scripts/lib/gateway-supervisor.sh /usr/local/lib/nemoclaw/gateway-supervisor.sh COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh +COPY --chown=root:root --chmod=0500 scripts/jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py COPY scripts/lib/clean_runtime_shell_env_shim.py /usr/local/lib/nemoclaw/clean_runtime_shell_env_shim.py COPY scripts/lib/normalize_mutable_config_perms.py /usr/local/lib/nemoclaw/normalize_mutable_config_perms.py @@ -2357,6 +2358,8 @@ RUN check_metadata() { \ && check_metadata /usr/local/bin/nemoclaw-managed-bootstrap 'root:root:755' \ && test ! -L /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh \ && check_metadata /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh 'root:root:444' \ + && test ! -L /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \ + && check_metadata /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh 'root:root:500' \ && check_metadata /usr/local/bin/nemoclaw-gateway-control 'root:root:700' \ && check_metadata /usr/local/lib/nemoclaw/preloads/sandbox-safety-net.js 'root:root:644' diff --git a/scripts/jetson-device-group-bootstrap.sh b/scripts/jetson-device-group-bootstrap.sh new file mode 100755 index 00000000000..12e662e5621 --- /dev/null +++ b/scripts/jetson-device-group-bootstrap.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Compatibility bridge for #7610. Remove when the minimum supported OpenShell +# release natively preserves Jetson device groups across the sandbox-user handoff. + +set -euo pipefail + +fail() { + printf 'Jetson device-group bootstrap: %s\n' "$*" >&2 + exit 1 +} + +[ "$(/usr/bin/id -u)" -eq 0 ] || fail "must run as root" +[ "${1:-}" = "--device-group-gids" ] || fail "device group argument is missing" +group_gids="${2:-}" +[ "${3:-}" = "--" ] || fail "supervisor delimiter is missing" +shift 3 +[ "${1:-}" = "/opt/openshell/bin/openshell-sandbox" ] \ + || fail "OpenShell supervisor entrypoint is invalid" +/usr/bin/id sandbox >/dev/null 2>&1 || fail "sandbox user is missing" +if [ ! -f /etc/group ] || [ -L /etc/group ]; then + fail "container group database is invalid" +fi + +IFS=',' read -r -a gids <<<"$group_gids" +if [ "${#gids[@]}" -eq 0 ] || [ "${#gids[@]}" -gt 16 ]; then + fail "device group count is invalid" +fi + +declare -A seen=() +for gid in "${gids[@]}"; do + [[ "$gid" =~ ^[1-9][0-9]{0,9}$ ]] || fail "device group ID is invalid" + [ "$gid" -le 2147483647 ] || fail "device group ID is out of range" + [ -z "${seen[$gid]:-}" ] || fail "device group ID is duplicated" + seen[$gid]=1 +done + +declare -a group_names=() +declare -a create_groups=() +for gid in "${gids[@]}"; do + group_record="$(/usr/bin/getent group "$gid" || true)" + if [ -z "$group_record" ]; then + group_name="nemoclaw_gpu_$gid" + [ -z "$(/usr/bin/getent group "$group_name" || true)" ] \ + || fail "generated device group name already exists" + create_group=1 + else + if [[ "$group_record" == *$'\n'* ]]; then + fail "device group record is invalid" + fi + IFS=':' read -r group_name _ resolved_gid _ extra <<<"$group_record" + if [ -z "$group_name" ] || [ "$resolved_gid" != "$gid" ] || [ -n "$extra" ]; then + fail "device group record is invalid" + fi + case "$group_name" in + video | render | "nemoclaw_gpu_$gid") ;; + *) fail "existing device group is not approved" ;; + esac + create_group=0 + fi + group_names+=("$group_name") + create_groups+=("$create_group") +done + +for index in "${!gids[@]}"; do + gid="${gids[$index]}" + group_name="${group_names[$index]}" + if [ "${create_groups[$index]}" -eq 1 ]; then + /usr/sbin/groupadd --gid "$gid" "$group_name" + fi + /usr/sbin/usermod --append --groups "$group_name" sandbox +done + +sandbox_groups=" $(/usr/bin/id -G sandbox) " +for gid in "${gids[@]}"; do + [[ "$sandbox_groups" == *" $gid "* ]] \ + || fail "sandbox membership verification failed" +done + +exec "$@" diff --git a/src/lib/onboard/docker-gpu-jetson-groups.test.ts b/src/lib/onboard/docker-gpu-jetson-groups.test.ts index 90b6dfb5446..d382e87da25 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.test.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.test.ts @@ -5,7 +5,56 @@ import fs from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { detectTegraDeviceGroupGids } from "./docker-gpu-jetson-groups"; +import { + detectTegraDeviceGroupGids, + detectTegraGpuDevicePaths, + JETSON_DEVICE_GROUP_BOOTSTRAP, + normalizeTegraGpuDevicePaths, + resolveJetsonDeviceGroupBootstrap, +} from "./docker-gpu-jetson-groups"; + +describe("detectTegraGpuDevicePaths", () => { + it("returns existing non-symlink character devices when nvmap anchors detection (#7610)", () => { + const paths = ["/dev/nvmap", "/dev/nvhost-gpu", "/dev/nvgpu/igpu0/link"]; + + expect( + detectTegraGpuDevicePaths({ + listDevicePaths: () => paths, + statDevicePath: (devicePath) => + devicePath === "/dev/nvmap" + ? { isCharacterDevice: true, isSymbolicLink: false } + : devicePath === "/dev/nvgpu/igpu0/link" + ? { isCharacterDevice: true, isSymbolicLink: true } + : null, + }), + ).toEqual(["/dev/nvmap"]); + }); + + it.each([ + null, + { isCharacterDevice: true, isSymbolicLink: true }, + { isCharacterDevice: false, isSymbolicLink: false }, + ])("returns no paths for invalid nvmap access case %# (#7610)", (nvmapAccess) => { + expect( + detectTegraGpuDevicePaths({ + listDevicePaths: () => ["/dev/nvmap", "/dev/dri/renderD128"], + statDevicePath: (devicePath) => + devicePath === "/dev/nvmap" + ? nvmapAccess + : { isCharacterDevice: true, isSymbolicLink: false }, + }), + ).toEqual([]); + }); + + it("rejects device-policy paths outside the bounded Tegra set (#7610)", () => { + expect(() => normalizeTegraGpuDevicePaths(["/dev/nvmap", "/dev/input/event0"])).toThrow( + "Jetson GPU device paths are invalid or excessive", + ); + expect( + normalizeTegraGpuDevicePaths(["/dev/dri/renderD128", "/dev/nvmap", "/dev/nvmap"]), + ).toEqual(["/dev/dri/renderD128", "/dev/nvmap"]); + }); +}); describe("detectTegraDeviceGroupGids", () => { afterEach(() => { @@ -65,6 +114,15 @@ describe("detectTegraDeviceGroupGids", () => { } }); + it("requires nvmap before accepting an unrelated render group (#7610)", () => { + expect( + detectTegraDeviceGroupGids({ + listDevicePaths: () => ["/dev/dri/renderD128"], + statDeviceAccess: () => ({ gid: 104, mode: 0o660 }), + }), + ).toEqual([]); + }); + it("rejects regular files and symlinks before reading their group", () => { const lstat = vi .spyOn(fs, "lstatSync") @@ -131,3 +189,43 @@ describe("detectTegraDeviceGroupGids", () => { } }); }); + +describe("resolveJetsonDeviceGroupBootstrap", () => { + const supervisorArgv = ["/opt/openshell/bin/openshell-sandbox", "--workdir", "/sandbox"]; + + it("wraps only the exact OpenClaw supervisor with normalized group IDs (#7610)", () => { + expect( + resolveJetsonDeviceGroupBootstrap({ + agent: "openclaw", + deviceGroupGids: ["44", "110"], + supervisorArgv, + }), + ).toEqual({ + deviceGroupGids: ["44", "110"], + supervisorArgv: [ + JETSON_DEVICE_GROUP_BOOTSTRAP, + "--device-group-gids", + "44,110", + "--", + ...supervisorArgv, + ], + }); + expect( + resolveJetsonDeviceGroupBootstrap({ + agent: "hermes", + deviceGroupGids: ["44"], + supervisorArgv, + }), + ).toBeNull(); + }); + + it("rejects a different OpenClaw launch authority (#7610)", () => { + expect(() => + resolveJetsonDeviceGroupBootstrap({ + agent: "openclaw", + deviceGroupGids: ["44"], + supervisorArgv: ["/tmp/openshell-sandbox"], + }), + ).toThrow("requires the OpenShell supervisor entrypoint"); + }); +}); diff --git a/src/lib/onboard/docker-gpu-jetson-groups.ts b/src/lib/onboard/docker-gpu-jetson-groups.ts index 4cfcd098f47..d193e83f0cd 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.ts @@ -16,14 +16,28 @@ const TEGRA_GPU_DEVICE_NODES = [ "/dev/nvgpu/igpu0/as", "/dev/nvgpu/igpu0/prof", ] as const; +const TEGRA_GPU_DEVICE_NODE_SET = new Set(TEGRA_GPU_DEVICE_NODES); +const NVMAP_DEVICE = "/dev/nvmap"; const READ_WRITE_PERMISSION_BITS = 0o6; const MAX_DOCKER_SUPPLEMENTARY_GID = 2_147_483_647; +const MAX_JETSON_DEVICE_GROUPS = 16; +const MAX_JETSON_GPU_DEVICE_PATHS = 32; + +export const NEMOCLAW_MANAGED_AGENT_LABEL = "io.nvidia.nemoclaw.agent"; +export const OPENSHELL_SANDBOX_ENTRYPOINT = "/opt/openshell/bin/openshell-sandbox"; +export const JETSON_DEVICE_GROUP_BOOTSTRAP = + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh"; type DeviceGroupAccess = { gid: number; mode: number; }; +type DevicePathAccess = { + isCharacterDevice: boolean; + isSymbolicLink: boolean; +}; + /** * Find real DRI render character devices without following symlinks or * scanning other DRI device families. @@ -51,6 +65,55 @@ function listTegraGpuDevicePaths(): string[] { return [...TEGRA_GPU_DEVICE_NODES, ...discoverTegraRenderDevicePaths()]; } +export function normalizeTegraGpuDevicePaths(devicePaths: readonly string[]): string[] { + const normalized = [...new Set(devicePaths)]; + if ( + normalized.length > MAX_JETSON_GPU_DEVICE_PATHS || + normalized.some( + (devicePath) => + devicePath.length > 64 || + (!TEGRA_GPU_DEVICE_NODE_SET.has(devicePath) && + !/^\/dev\/dri\/renderD\d+$/u.test(devicePath)), + ) + ) { + throw new Error("Jetson GPU device paths are invalid or excessive."); + } + return normalized.includes(NVMAP_DEVICE) ? normalized : []; +} + +/** + * Return the fixed Jetson GPU device paths that are safe to add to an + * OpenShell filesystem policy. `/dev/nvmap` anchors detection so a host with + * only an unrelated DRI render device does not receive Jetson grants. + */ +export function detectTegraGpuDevicePaths( + deps: { + statDevicePath?: (path: string) => DevicePathAccess | null; + listDevicePaths?: () => string[]; + } = {}, +): string[] { + const devicePaths = deps.listDevicePaths?.() ?? listTegraGpuDevicePaths(); + const statDevicePath = + deps.statDevicePath ?? + ((devicePath: string): DevicePathAccess | null => { + try { + const stat = fs.lstatSync(devicePath); + return { + isCharacterDevice: stat.isCharacterDevice(), + isSymbolicLink: stat.isSymbolicLink(), + }; + } catch { + return null; + } + }); + + const detectedPaths = devicePaths.filter((devicePath) => { + const access = statDevicePath(devicePath); + return access?.isCharacterDevice === true && access.isSymbolicLink === false; + }); + return normalizeTegraGpuDevicePaths(detectedPaths); +} + /** * Source-of-truth boundary for Jetson/Tegra supplementary device groups: * @@ -86,9 +149,14 @@ export function detectTegraDeviceGroupGids( return null; } }); + const deviceAccess = new Map( + devicePaths.map((devicePath) => [devicePath, statAccess(devicePath)] as const), + ); + if (deviceAccess.get(NVMAP_DEVICE) == null) return []; + const gids = new Set(); for (const node of devicePaths) { - const access = statAccess(node); + const access = deviceAccess.get(node) ?? null; const gid = access?.gid ?? null; const groupAccessBits = access === null ? 0 : (access.mode >> 3) & READ_WRITE_PERMISSION_BITS; const otherAccessBits = access === null ? 0 : access.mode & READ_WRITE_PERMISSION_BITS; @@ -105,3 +173,51 @@ export function detectTegraDeviceGroupGids( } return [...gids].sort((left, right) => Number(left) - Number(right)); } + +export function normalizeJetsonDeviceGroupGids(groupGids: readonly string[]): string[] { + const normalized = groupGids.map((gid) => String(gid).trim()); + if ( + normalized.length > MAX_JETSON_DEVICE_GROUPS || + new Set(normalized).size !== normalized.length || + normalized.some((gid) => { + if (!/^[1-9][0-9]*$/u.test(gid)) return true; + const parsed = Number(gid); + return !Number.isSafeInteger(parsed) || parsed > MAX_DOCKER_SUPPLEMENTARY_GID; + }) + ) { + throw new Error("Docker clone received invalid or excessive supplementary group IDs."); + } + return normalized; +} + +export type JetsonDeviceGroupBootstrapPlan = Readonly<{ + deviceGroupGids: readonly string[]; + supervisorArgv: readonly string[]; +}>; + +/** + * Render the only approved root bootstrap handoff for an OpenClaw managed + * image. The exact agent identity and supervisor entrypoint keep this launch + * authority closed to other images and commands. + */ +export function resolveJetsonDeviceGroupBootstrap(input: { + agent: string | null | undefined; + deviceGroupGids: readonly string[]; + supervisorArgv: readonly string[]; +}): JetsonDeviceGroupBootstrapPlan | null { + const deviceGroupGids = Object.freeze(normalizeJetsonDeviceGroupGids(input.deviceGroupGids)); + if (input.agent !== "openclaw" || deviceGroupGids.length === 0) return null; + if (input.supervisorArgv[0] !== OPENSHELL_SANDBOX_ENTRYPOINT) { + throw new Error("Jetson device-group bootstrap requires the OpenShell supervisor entrypoint."); + } + return Object.freeze({ + deviceGroupGids, + supervisorArgv: Object.freeze([ + JETSON_DEVICE_GROUP_BOOTSTRAP, + "--device-group-gids", + deviceGroupGids.join(","), + "--", + ...input.supervisorArgv, + ]), + }); +} diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index 578159285fb..bb7f0fce284 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -7,10 +7,15 @@ import type { DockerGpuPatchMode, DockerUlimit, } from "./docker-gpu-patch-types"; +import { + NEMOCLAW_MANAGED_AGENT_LABEL, + normalizeJetsonDeviceGroupGids, + OPENSHELL_SANDBOX_ENTRYPOINT, + resolveJetsonDeviceGroupBootstrap, +} from "./docker-gpu-jetson-groups"; import { openshellSandboxCommandEnvValue } from "./docker-startup-command-env"; const OPENSHELL_SANDBOX_COMMAND_ENV = "OPENSHELL_SANDBOX_COMMAND"; -const OPENSHELL_SANDBOX_ENTRYPOINT = "/opt/openshell/bin/openshell-sandbox"; const OPENSHELL_V0_0_99_WORKDIR_COMMAND = ["--workdir", "/sandbox"] as const; const OPENSHELL_OCI_IMAGE_USER_ENV = "OPENSHELL_OCI_IMAGE_USER"; const OPENSHELL_SANDBOX_UID_ENV = "OPENSHELL_SANDBOX_UID"; @@ -28,6 +33,8 @@ type DockerStructuredMount = NonNullable< >[number]; export const DOCKER_GPU_PATCH_NETWORK_ENV = "NEMOCLAW_DOCKER_GPU_PATCH_NETWORK"; +export const MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE = + "/usr/local/bin/nemoclaw-managed-bootstrap"; export function dockerContainerName(inspect: DockerContainerInspect): string { const raw = String(inspect.Name || "") @@ -582,6 +589,7 @@ export function buildDockerGpuCloneRunArgs( ): string[] { const config = inspect.Config || {}; const host = inspect.HostConfig || {}; + const labels = config.Labels || {}; const image = String(options.image || config.Image || "").trim(); if (!image) throw new Error("Docker inspect output did not include Config.Image."); @@ -595,6 +603,7 @@ export function buildDockerGpuCloneRunArgs( } const args: string[] = ["--name", containerName, ...mode.args]; const gpuAugment = mode.kind !== "startup-command"; + const extraGroupGids = normalizeJetsonDeviceGroupGids(options.extraGroupGids ?? []); // Startup-command recreation must retain OpenShell's native CDI attachment. if (!gpuAugment) { @@ -669,7 +678,6 @@ export function buildDockerGpuCloneRunArgs( const value = annotations[key]; if (value !== undefined && value !== null) args.push("--annotation", `${key}=${value}`); } - const labels = config.Labels || {}; for (const key of Object.keys(labels).sort()) { const value = labels[key]; if (value !== undefined && value !== null) args.push("--label", `${key}=${value}`); @@ -705,11 +713,10 @@ export function buildDockerGpuCloneRunArgs( for (const hostEntry of stringArray(host.ExtraHosts)) args.push("--add-host", hostEntry); const groupAdds = new Set(stringArray(host.GroupAdd)); for (const group of groupAdds) args.push("--group-add", group); - for (const gid of options.extraGroupGids ?? []) { - const normalized = String(gid).trim(); - if (normalized && !groupAdds.has(normalized)) { - groupAdds.add(normalized); - args.push("--group-add", normalized); + for (const gid of extraGroupGids) { + if (!groupAdds.has(gid)) { + groupAdds.add(gid); + args.push("--group-add", gid); } } for (const ulimit of dockerUlimits(inspect, options.requiredUlimits)) { @@ -743,17 +750,26 @@ export function buildDockerGpuCloneRunArgs( const entrypoint = stringArray(config.Entrypoint); const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim(); - if (replacementEntrypoint) { - args.push("--entrypoint", replacementEntrypoint); - } else if (entrypoint.length > 0) { - args.push("--entrypoint", entrypoint[0]); - } - const commandArgs = dockerContainerCommandArgs( + const groupBootstrapTarget = replacementEntrypoint || entrypoint[0] || ""; + const targetCommandArgs = dockerContainerCommandArgs( entrypoint, stringArray(config.Cmd), sandboxCommand, options.containerCommand, ); + const bootstrapPlan = + replacementEntrypoint === MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE + ? null + : resolveJetsonDeviceGroupBootstrap({ + agent: labels[NEMOCLAW_MANAGED_AGENT_LABEL], + deviceGroupGids: extraGroupGids, + supervisorArgv: [groupBootstrapTarget, ...targetCommandArgs], + }); + const [selectedEntrypoint, ...commandArgs] = bootstrapPlan?.supervisorArgv ?? [ + groupBootstrapTarget, + ...targetCommandArgs, + ]; + if (selectedEntrypoint) args.push("--entrypoint", selectedEntrypoint); args.push(image, ...commandArgs); return args; } diff --git a/src/lib/onboard/docker-gpu-patch-jetson.test.ts b/src/lib/onboard/docker-gpu-patch-jetson.test.ts index fc6c769e70b..046cc26272e 100644 --- a/src/lib/onboard/docker-gpu-patch-jetson.test.ts +++ b/src/lib/onboard/docker-gpu-patch-jetson.test.ts @@ -10,10 +10,16 @@ import { recreateOpenShellDockerSandboxWithGpu, } from "./docker-gpu-patch"; -function dockerCaptureFixture() { +function openClawInspectFixture() { + const inspect = inspectFixture(); + inspect.Config!.Labels!["io.nvidia.nemoclaw.agent"] = "openclaw"; + return inspect; +} + +function dockerCaptureFixture(inspect = inspectFixture()) { const responses: Record = { ps: "old-container-id\n", - inspect: JSON.stringify([inspectFixture()]), + inspect: JSON.stringify([inspect]), info: "", }; return vi.fn((args: readonly string[]) => responses[args[0]] ?? ""); @@ -41,6 +47,50 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect(args).not.toEqual(expect.arrayContaining(["--group-add"])); }); + it("wraps only the fixed OpenClaw supervisor with validated device groups (#7610)", () => { + const args = buildDockerGpuCloneRunArgs( + openClawInspectFixture(), + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { extraGroupGids: ["44"] }, + ); + + expect(args.slice(args.indexOf("openshell/sandbox:abc"))).toEqual([ + "openshell/sandbox:abc", + "--device-group-gids", + "44", + "--", + "/opt/openshell/bin/openshell-sandbox", + ]); + }); + + it.each([ + ["0"], + ["2147483648"], + ["44", "44"], + ...[Array.from({ length: 17 }, (_, index) => String(index + 1))], + ])("rejects invalid or excessive supplementary groups case %# (#7610)", (...gids) => { + expect(() => + buildDockerGpuCloneRunArgs( + openClawInspectFixture(), + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { extraGroupGids: gids }, + ), + ).toThrow("invalid or excessive supplementary group IDs"); + }); + + it("rejects an OpenClaw bootstrap outside the fixed supervisor boundary (#7610)", () => { + const inspect = openClawInspectFixture(); + inspect.Config!.Entrypoint = ["/custom/entrypoint"]; + + expect(() => + buildDockerGpuCloneRunArgs( + inspect, + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { extraGroupGids: ["44"] }, + ), + ).toThrow("requires the OpenShell supervisor entrypoint"); + }); + it("passes all detected Tegra device GIDs into the Jetson recreate as --group-add", () => { const dockerRunDetached = vi.fn(() => ({ status: 0, @@ -112,4 +162,28 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect.anything(), ); }); + + it("keeps the original OpenClaw container running when the helper is missing (#7610)", () => { + const dockerStop = vi.fn(() => ({ status: 0 })); + + expect(() => + recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1, backend: "jetson" }, + { + dockerCapture: dockerCaptureFixture(openClawInspectFixture()), + dockerRun: vi.fn((args: readonly string[]) => ({ status: args[0] === "exec" ? 1 : 0 })), + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop, + dockerRm: vi.fn(() => ({ status: 0 })), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-15T00:00:00Z"), + detectSandboxFallbackDns: () => null, + detectTegraDeviceGroupGids: () => ["44"], + }, + ), + ).toThrow("OpenClaw sandbox image is missing executable"); + expect(dockerStop).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index 7d2310ff366..0d5befb704e 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -12,7 +12,11 @@ import { } from "../adapters/docker"; import { hasZeroDockerExitStatus } from "./docker-command-result"; import { detectSandboxFallbackDns } from "./docker-gpu-dns-fallback"; -import { detectTegraDeviceGroupGids } from "./docker-gpu-jetson-groups"; +import { + detectTegraDeviceGroupGids, + JETSON_DEVICE_GROUP_BOOTSTRAP, + NEMOCLAW_MANAGED_AGENT_LABEL, +} from "./docker-gpu-jetson-groups"; import { buildDockerGpuCloneRunArgs, buildDockerGpuCloneRunOptions, @@ -262,15 +266,43 @@ export function recreateOpenShellDockerSandboxContainer( ); } } - if (selection.mode.kind !== "startup-command" && options.backend === "jetson") { + const managedAgent = inspect.Config?.Labels?.[NEMOCLAW_MANAGED_AGENT_LABEL] ?? null; + const shouldBootstrapJetsonGroups = managedAgent === "openclaw"; + const shouldDetectJetsonGroups = + options.backend === "jetson" && + (selection.mode.kind !== "startup-command" || shouldBootstrapJetsonGroups); + if (shouldDetectJetsonGroups) { const tegraGroupGids = d.detectTegraDeviceGroupGids(); if (tegraGroupGids.length > 0) { cloneOptions.extraGroupGids = tegraGroupGids; - console.log( - ` ✓ Granting sandbox user the detected Jetson GPU device groups via --group-add ${tegraGroupGids.join( - ", ", - )} (so CUDA can initialize as a non-root user)`, - ); + if (shouldBootstrapJetsonGroups) { + const wrapperProbe = d.dockerRun( + [ + "exec", + "--user", + "0", + oldContainerId, + "/usr/bin/test", + "-x", + JETSON_DEVICE_GROUP_BOOTSTRAP, + ], + { ignoreError: true, suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS }, + ); + if (!hasZeroDockerExitStatus(wrapperProbe)) { + throw new Error( + `OpenClaw sandbox image is missing executable ${JETSON_DEVICE_GROUP_BOOTSTRAP}.`, + ); + } + console.log( + ` ✓ Preserving the detected Jetson GPU device groups through OpenShell startup: ${tegraGroupGids.join(", ")}`, + ); + } else { + console.log( + ` ✓ Granting sandbox user the detected Jetson GPU device groups via --group-add ${tegraGroupGids.join( + ", ", + )} (so CUDA can initialize as a non-root user)`, + ); + } } else { console.warn( " ⚠ Could not resolve the group owning Jetson Tegra GPU device nodes (/dev/nvmap); CUDA may fail with NvRmMemInitNvmap permission denied. Confirm /dev/nvmap exists and is group-readable on the host.", diff --git a/src/lib/onboard/docker-startup-command-agent.ts b/src/lib/onboard/docker-startup-command-agent.ts index 6f3c3200e15..77fe9df9c7c 100644 --- a/src/lib/onboard/docker-startup-command-agent.ts +++ b/src/lib/onboard/docker-startup-command-agent.ts @@ -23,11 +23,14 @@ export function resolveDockerStartupCommandPatch( persistStartupCommand: boolean; requiredUlimits: readonly DockerUlimit[] | null; } { - if (dockerDriverGateway !== true) { - return { persistStartupCommand: false, requiredUlimits: null }; - } const agentName = agent?.name ?? "openclaw"; const requiredUlimits = agentName === DCODE_AGENT_NAME ? DCODE_DOCKER_ULIMITS : null; + if (dockerDriverGateway !== true) { + return { + persistStartupCommand: false, + requiredUlimits: null, + }; + } // The restart-safe recreation discovers the sandbox with docker-driver // labels (openshell.ai/managed-by), but the portable profile registers the // gateway with the podman driver, whose containers never carry that label — diff --git a/src/lib/onboard/docker-startup-command-patch.test.ts b/src/lib/onboard/docker-startup-command-patch.test.ts index 8923bb53062..6bafb9c9b4b 100644 --- a/src/lib/onboard/docker-startup-command-patch.test.ts +++ b/src/lib/onboard/docker-startup-command-patch.test.ts @@ -101,6 +101,60 @@ describe("Docker startup-command patch", () => { ); }); + it("preserves Jetson device groups during startup-command recreation (#7610)", () => { + const inspect = inspectFixture(); + inspect.Config!.Labels!["io.nvidia.nemoclaw.agent"] = "openclaw"; + const dockerCaptureOutput: Record = { + ps: "old-container-id\n", + inspect: JSON.stringify([inspect]), + }; + const dockerRunDetached = vi.fn((_args: readonly string[]) => ({ + status: 0, + stdout: "new-container-id\n", + })); + + recreateStartupCommandForTest( + { + sandboxName: "alpha", + timeoutSecs: 1, + waitForSupervisor: false, + openshellSandboxCommand: ["nemoclaw-start"], + backend: "jetson", + }, + { + dockerCapture: vi.fn((args: readonly string[]) => dockerCaptureOutput[args[0] ?? ""] ?? ""), + dockerRun: vi.fn(() => ({ status: 0 })), + dockerRunDetached, + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-07-10T00:00:00Z"), + detectTegraDeviceGroupGids: () => ["44", "993"], + }, + ); + + const cloneArgs = dockerRunDetached.mock.calls[0]?.[0] ?? []; + expect(cloneArgs).toEqual( + expect.arrayContaining([ + "--group-add", + "44", + "--group-add", + "993", + "--entrypoint", + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", + ]), + ); + expect(cloneArgs.slice(cloneArgs.indexOf(`sha256:${"c".repeat(64)}`))).toEqual([ + `sha256:${"c".repeat(64)}`, + "--device-group-gids", + "44,993", + "--", + "/opt/openshell/bin/openshell-sandbox", + "--workdir", + "/sandbox", + ]); + }); + it("preserves OpenShell's native CDI GPU request during restart-persistence recreation", () => { const inspect = inspectFixture(); inspect.HostConfig!.DeviceRequests = [ @@ -211,39 +265,39 @@ describe("Docker startup-command patch", () => { expect(dockerRunDetached).not.toHaveBeenCalled(); }); - it.each([ - "different-container-id", - "", - ])("refuses to mutate when the pinned container identity is changed or empty", (expectedOldContainerId) => { - const dockerStop = vi.fn(() => ({ status: 0 })); - const dockerRename = vi.fn(() => ({ status: 0 })); - const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + it.each(["different-container-id", ""])( + "refuses to mutate when the pinned container identity is changed or empty", + (expectedOldContainerId) => { + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); - expect(() => - recreateStartupCommandForTest( - { - sandboxName: "alpha", - openshellSandboxCommand: ["env", "nemoclaw-start"], - expectedOldContainerId, - }, - { - dockerCapture: vi.fn((args: readonly string[]) => - args[0] === "ps" - ? "old-container-id\n" - : args[0] === "inspect" - ? JSON.stringify([inspectFixture()]) - : "", - ), - dockerRunDetached, - dockerRename, - dockerStop, - }, - ), - ).toThrow("observed container differs from the pinned identity"); - expect(dockerStop).not.toHaveBeenCalled(); - expect(dockerRename).not.toHaveBeenCalled(); - expect(dockerRunDetached).not.toHaveBeenCalled(); - }); + expect(() => + recreateStartupCommandForTest( + { + sandboxName: "alpha", + openshellSandboxCommand: ["env", "nemoclaw-start"], + expectedOldContainerId, + }, + { + dockerCapture: vi.fn((args: readonly string[]) => + args[0] === "ps" + ? "old-container-id\n" + : args[0] === "inspect" + ? JSON.stringify([inspectFixture()]) + : "", + ), + dockerRunDetached, + dockerRename, + dockerStop, + }, + ), + ).toThrow("observed container differs from the pinned identity"); + expect(dockerStop).not.toHaveBeenCalled(); + expect(dockerRename).not.toHaveBeenCalled(); + expect(dockerRunDetached).not.toHaveBeenCalled(); + }, + ); it("does not rename or recreate when the original container cannot be stopped", () => { const dockerRename = vi.fn(() => ({ status: 0 })); diff --git a/src/lib/onboard/docker-startup-command-patch.ts b/src/lib/onboard/docker-startup-command-patch.ts index 2b5b6f761ef..509781463fc 100644 --- a/src/lib/onboard/docker-startup-command-patch.ts +++ b/src/lib/onboard/docker-startup-command-patch.ts @@ -18,6 +18,7 @@ export function recreateOpenShellDockerSandboxWithStartupCommand( openshellSandboxCommand: readonly string[]; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; + backend?: import("./docker-gpu-patch-types").DockerGpuPatchBackend; }, deps: DockerGpuPatchDeps = {}, ): DockerGpuPatchResult { diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.ts b/src/lib/onboard/docker-startup-command-sandbox-create.ts index dd8df2e17b0..e76e1acb6eb 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.ts @@ -33,6 +33,7 @@ export function createDockerSandboxRecreator(options: { requiredUlimits: options.requiredUlimits, timeoutSecs: options.gpuOptions.timeoutSecs, waitForSupervisor, + backend: options.gpuOptions.backend, }, deps, ); diff --git a/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts b/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts index abd2b99374a..b8af15a72af 100644 --- a/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts +++ b/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts @@ -57,6 +57,7 @@ const CANONICAL_POST_GENERATOR_INSTRUCTION_SHA256 = new Set([ "a0a554d474cb70087e50686d998915eae06201d6182a2410d3ccc4879e5058e6", "5af905889f94ffed2f6c371111d0589e38eed7b0de54ddb0dd68ad912a23149a", "1197b99bdb996b37a3e4e386a507dfabcdfb2c26a40b015d617f97208668187d", + "40f0276b5d7359bb0396448eb600fbdfe491f0f923fb4bd54198f063c0ad5875", "3d5708120490fab5a69d1e592252cc0e2a79a6ea1b963f714dad8a6e971e4937", "c0b409e1bf4d33a9e44f407c6bd9b0445b2ffd0b796823fe3cfa5989314d6603", "9fcc674a44a152707380cdb09a67f8594f568288406c96f5354f1c87f5b939a6", @@ -76,6 +77,7 @@ const CANONICAL_POST_GENERATOR_INSTRUCTION_SHA256 = new Set([ "8f5966da093ef75cefd35c2b7f1361fbf5b32e63a4a8a34cb3ac7f76a1330e5e", "c6b042ac2cc3d5570ae43f1e387a951cbe89fbb22e8cd9df486f99719bb32939", "ba29b499af923b4331cf7abf14648f187dc0e0b8f3dc2c33dac61f079981c187", + "f91353eff1014dbe84120fc954cf2db18be7c3b01b3af95f6bbc0dbb2771003d", "9c39e3aefbfefb7a48c6ba6879701954ad300f3593d18713063a12056f71af93", // COPY --from=openclaw-runtime-payload / / // The reviewed scratch payload has no /sandbox/.openclaw content, so this diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index a8395415065..5b8aa8a8df2 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -5,7 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest"; import YAML from "yaml"; const logPresetScopeMock = vi.hoisted(() => vi.fn()); @@ -520,6 +520,70 @@ network_policies: {} ); }); + it("moves detected Jetson devices to read-write and adds libraries as read-only (#7610)", () => { + const gpuPolicy = buildDirectGpuPolicyYaml( + ` +version: 1 +filesystem_policy: + read_only: + - /usr + - /dev/nvmap + read_write: + - /tmp + - /dev/nvhost-gpu +network_policies: {} +`, + { jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvhost-gpu", "/dev/nvmap"] }, + ); + const gpuDoc = YAML.parse(gpuPolicy); + + expect(gpuDoc.filesystem_policy.read_only).toContain("/opt/nvidia"); + expect(gpuDoc.filesystem_policy.read_only).not.toContain("/dev/nvmap"); + expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvmap"); + expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvhost-gpu"); + }); + + it("omits Jetson grants when no device paths are detected (#7610)", () => { + const gpuDoc = YAML.parse(buildDirectGpuPolicyYaml(BASE_POLICY_FIXTURE)); + + expect(gpuDoc.filesystem_policy.read_only).not.toContain("/opt/nvidia"); + expect(gpuDoc.filesystem_policy.read_write).not.toContain("/opt/nvidia"); + }); + + it("keeps Jetson filesystem grants scoped to OpenClaw direct GPU policy (#7610)", () => { + const basePolicyPath = tmpPolicy(BASE_POLICY_FIXTURE); + const devicePaths = ["/dev/nvmap", "/dev/nvhost-gpu"]; + const defaultOpenclaw = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { + directGpu: true, + jetsonGpuDevicePaths: devicePaths, + stationGb300SysfsReadOnlyPaths: [], + }); + const hermes = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { + directGpu: true, + agentName: "hermes", + jetsonGpuDevicePaths: devicePaths, + stationGb300SysfsReadOnlyPaths: [], + }); + onTestFinished(() => { + const defaultOpenclawCleaned = defaultOpenclaw.cleanup?.(); + const hermesCleaned = hermes.cleanup?.(); + expect(defaultOpenclawCleaned).toBe(true); + expect(hermesCleaned).toBe(true); + }); + const defaultOpenclawDoc = YAML.parse(fs.readFileSync(defaultOpenclaw.policyPath, "utf-8")); + const hermesDoc = YAML.parse(fs.readFileSync(hermes.policyPath, "utf-8")); + + expect(defaultOpenclawDoc.filesystem_policy.read_only).toContain("/opt/nvidia"); + expect(defaultOpenclawDoc.filesystem_policy.read_write).toEqual( + expect.arrayContaining(devicePaths), + ); + expect(hermesDoc.filesystem_policy.read_only).not.toContain("/opt/nvidia"); + expect(hermesDoc.filesystem_policy.read_write).not.toContain("/dev/nvmap"); + expect(hermesDoc.filesystem_policy.read_write).not.toContain("/dev/nvhost-gpu"); + expect(hermesDoc.filesystem_policy.read_only).not.toContain("/dev/nvmap"); + expect(hermesDoc.filesystem_policy.read_only).not.toContain("/dev/nvhost-gpu"); + }); + it.each(Array.from(STATION_GB300_SYSFS_READ_ONLY_PATHS, (value) => [value]))( "keeps %s outside the non-Station direct GPU policy (#7103)", (sysfsPath) => { diff --git a/src/lib/onboard/initial-policy.ts b/src/lib/onboard/initial-policy.ts index 0226d39b19e..7937526ab8c 100644 --- a/src/lib/onboard/initial-policy.ts +++ b/src/lib/onboard/initial-policy.ts @@ -21,6 +21,10 @@ import { isStationGb300ProductName, type StationProfile, } from "../readiness/station-qualification"; +import { + detectTegraGpuDevicePaths, + normalizeTegraGpuDevicePaths, +} from "./docker-gpu-jetson-groups"; import { allMessagingChannelPolicyPresets, requiredMessagingChannelPolicyPresets, @@ -51,6 +55,7 @@ export function discloseInitialSandboxPolicy(policy: InitialSandboxPolicy): void const HERMES_MESSAGING_POLICY_KEYS = getMessagingPolicyKeysByChannel({ agent: "hermes" }); const PROC_PATH = "/proc"; +const JETSON_GPU_LIBRARY_ROOT = "/opt/nvidia"; const PROC_COMM_READ_WRITE_PATHS = ["/proc/self/comm", "/proc/self/task/*/comm"]; const SYSFS_PATH = "/sys"; const PCI_BDF_PATTERN = /^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$/iu; @@ -82,6 +87,7 @@ function deduplicateDirectGpuSysfsEntries( type DirectGpuPolicyOptions = { procReadWrite?: boolean; sysfsReadOnlyPaths?: readonly string[]; + jetsonGpuDevicePaths?: readonly string[]; }; export { isStationGb300ProductName }; @@ -228,6 +234,23 @@ export function buildDirectGpuPolicyYaml( } } } + const jetsonGpuDevicePaths = normalizeTegraGpuDevicePaths(options.jetsonGpuDevicePaths ?? []); + if (jetsonGpuDevicePaths.length > 0) { + if ( + !fsPolicy.read_only.includes(JETSON_GPU_LIBRARY_ROOT) && + !fsPolicy.read_write.includes(JETSON_GPU_LIBRARY_ROOT) + ) { + fsPolicy.read_only.push(JETSON_GPU_LIBRARY_ROOT); + } + + const jetsonGpuDevicePathSet = new Set(jetsonGpuDevicePaths); + fsPolicy.read_only = fsPolicy.read_only.filter( + (entry: string) => !jetsonGpuDevicePathSet.has(entry), + ); + for (const devicePath of jetsonGpuDevicePaths) { + if (!fsPolicy.read_write.includes(devicePath)) fsPolicy.read_write.push(devicePath); + } + } if (options.procReadWrite && !fsPolicy.read_write.includes(PROC_PATH)) { // This exists only for the legacy post-create Docker GPU compatibility // path, which recreates the container after `openshell sandbox create` and @@ -319,6 +342,7 @@ type InitialPolicyOptions = { dockerGpuPatch?: boolean; hostGpuAvailable?: boolean; stationGb300SysfsReadOnlyPaths?: readonly string[]; + jetsonGpuDevicePaths?: readonly string[]; additionalPresets?: string[]; agentName?: string | null; sandboxName?: string; @@ -454,6 +478,10 @@ function resolveInitialSandboxCreatePolicy( discoverHostStationGb300SysfsReadOnlyPaths({ hasNvidiaGpu: options.hostGpuAvailable, }), + jetsonGpuDevicePaths: + (options.agentName ?? "openclaw") === "openclaw" + ? (options.jetsonGpuDevicePaths ?? detectTegraGpuDevicePaths()) + : [], }), "nemoclaw-gpu-policy", ); diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts index 48a3ddf9ee2..b95ae69cf60 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts @@ -26,6 +26,9 @@ const adapterMocks = vi.hoisted(() => ({ finalize: vi.fn(), prepare: vi.fn(), })); +const jetsonMocks = vi.hoisted(() => ({ + detectTegraDeviceGroupGids: vi.fn(() => ["44", "110"]), +})); const runtimeSnapshotMocks = vi.hoisted(() => ({ query: vi.fn(), })); @@ -57,6 +60,11 @@ vi.mock("./adapter", async (importOriginal) => ({ prepareManagedBootstrapSequence: adapterMocks.prepare, })); +vi.mock("../docker-gpu-jetson-groups", async (importOriginal) => ({ + ...(await importOriginal()), + detectTegraDeviceGroupGids: jetsonMocks.detectTegraDeviceGroupGids, +})); + vi.mock("../docker-gpu-sandbox-create", async (importOriginal) => ({ ...(await importOriginal()), isDockerDesktopWslRuntime: sandboxCreateMocks.isDockerDesktopWslRuntime, @@ -184,6 +192,7 @@ function gpuModeDependencies() { beforeEach(() => { vi.clearAllMocks(); + jetsonMocks.detectTegraDeviceGroupGids.mockReturnValue(["44", "110"]); dockerAdapterMocks.imageInspect.mockReturnValue({ status: 0 }); dockerAdapterMocks.pullWithProgressWatchdog.mockResolvedValue({ status: 0, @@ -496,6 +505,27 @@ describe("Docker managed-bootstrap GPU probe image", () => { }); describe("Docker managed-bootstrap lifecycle composition", () => { + it("adds detected Jetson groups to an OpenClaw native replacement (#7610)", async () => { + const seed = authority("openclaw"); + const { dependencies } = gpuModeDependencies(); + const input = compatibilityLifecycleInput(seed, dependencies); + + await runCompatibilityCreate( + { + ...input, + route: "native", + persistStartupCommand: true, + sandboxGpuConfig: { ...input.sandboxGpuConfig, hostGpuPlatform: "jetson" }, + }, + seed, + ); + + expect(adapterMocks.prepare.mock.calls[0]?.[1].replacementOptions.values).toMatchObject({ + extraGroupGids: ["44", "110"], + }); + expect(jetsonMocks.detectTegraDeviceGroupGids).toHaveBeenCalledOnce(); + }); + it("activates a Ready managed hold before post-activation startup output", async () => { vi.useFakeTimers(); const stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-runtime-")); diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts index b4e028e5dc3..2d9362707da 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -60,6 +60,8 @@ function dockerReplacementOptions( input: ManagedBootstrapRuntimeCreateLifecycleInput, ) { const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + const includeJetsonDeviceGroups = + backend === "jetson" && (input.route === "compatibility" || input.request.agent === "openclaw"); return { values: { gpuModeArgs: [...mode.args], @@ -69,8 +71,7 @@ function dockerReplacementOptions( requiredUlimits: input.requiredLimits.map( (limit) => `${limit.name}=${limit.soft}:${limit.hard}`, ), - extraGroupGids: - backend === "jetson" && input.route === "compatibility" ? detectTegraDeviceGroupGids() : [], + extraGroupGids: includeJetsonDeviceGroups ? detectTegraDeviceGroupGids() : [], }, }; } diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 4cabe3df064..ae623aec371 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -6,6 +6,7 @@ import fs from "node:fs"; import { expect, vi } from "vitest"; import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import { NEMOCLAW_MANAGED_AGENT_LABEL } from "../docker-gpu-jetson-groups"; import type { DockerContainerInspect } from "../docker-gpu-patch-types"; import { encodeManagedStartupProfile, type ManagedStartupAgent } from "../managed-startup/profile"; import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; @@ -81,6 +82,7 @@ export type DockerFixtureOptions = { Readonly> >; readonly lostAcknowledgements?: readonly DockerFixtureAcknowledgement[]; + readonly managedImageAgentLabel?: ManagedStartupAgent; readonly ownerId?: string; readonly replacementEnvironment?: (environment: readonly string[]) => readonly string[]; readonly sharedState?: "committed" | "none" | "pending"; @@ -120,7 +122,10 @@ export const sandbox = { driverId: "docker", }; -function originalInspect(inputs = agentInputs()): DockerContainerInspect { +function originalInspect( + inputs = agentInputs(), + managedImageAgentLabel?: ManagedStartupAgent, +): DockerContainerInspect { return { Id: OLD_ID, Image: CONFIG_ID, @@ -139,6 +144,9 @@ function originalInspect(inputs = agentInputs()): DockerContainerInspect { "openshell.ai/managed-by": "openshell", "openshell.ai/sandbox-name": "alpha", "openshell.ai/sandbox-id": "sandbox-alpha", + ...(managedImageAgentLabel + ? { [NEMOCLAW_MANAGED_AGENT_LABEL]: managedImageAgentLabel } + : {}), ...inputs.metadata, }, Entrypoint: [SUPERVISOR[0]], @@ -153,6 +161,7 @@ function originalInspect(inputs = agentInputs()): DockerContainerInspect { NetworkMode: "openshell", RestartPolicy: { Name: "unless-stopped" }, CapDrop: ["NET_RAW"], + GroupAdd: [], SecurityOpt: ["no-new-privileges"], Ulimits: [{ Name: "nofile", Soft: 65_536, Hard: 65_536 }], }, @@ -220,7 +229,10 @@ export function parseFixtureDockerUlimits( } export function fixture(options: DockerFixtureOptions = {}) { - let original: DockerContainerInspect | null = originalInspect(agentInputs(options.agent)); + let original: DockerContainerInspect | null = originalInspect( + agentInputs(options.agent), + options.managedImageAgentLabel, + ); if (options.dockerCliSerializationDefaults) { Object.assign(original.Config!, { AttachStdin: false, @@ -366,6 +378,9 @@ export function fixture(options: DockerFixtureOptions = {}) { const env = dockerOptions.flatMap((value, index) => value === "--env" ? [String(args[index + 1] ?? "")] : [], ); + const groupAdd = dockerOptions.flatMap((value, index) => + value === "--group-add" ? [String(args[index + 1] ?? "")] : [], + ); const ulimits = parseFixtureDockerUlimits(args, imageIndex); replacement = { ...structuredClone(source), @@ -380,6 +395,7 @@ export function fixture(options: DockerFixtureOptions = {}) { }, HostConfig: { ...structuredClone(source.HostConfig), + ...(groupAdd.length > 0 ? { GroupAdd: groupAdd } : {}), Ulimits: ulimits, }, State: { Running: false, Paused: false, Restarting: false, Dead: false }, diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 2f51469ce35..cfe51c60ab7 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -2,7 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import { assert, describe, expect, it, vi } from "vitest"; - +import { JETSON_DEVICE_GROUP_BOOTSTRAP } from "../docker-gpu-jetson-groups"; +import { MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE } from "../docker-gpu-patch-clone"; import { MANAGED_BOOTSTRAP_IDENTITY_ENV, ManagedBootstrapDurableCommitCleanupPendingError, @@ -217,6 +218,76 @@ describe("Docker managed bootstrap adapter", () => { ]); }); + it("preserves Jetson device groups through the managed supervisor handoff (#7610)", async () => { + const fake = fixture({ agent: "openclaw", managedImageAgentLabel: "openclaw" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request } = authority("openclaw"); + const discovered = await adapter.discoverHeldWorkload({ + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + expectedImage: handle.plan.image, + metadata: handle.plan.metadata, + }); + const snapshot = await adapter.inspectHeldWorkload({ handle, discovered }); + + await expect( + adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions: { + values: { + extraGroupGids: ["44", "110"], + }, + }, + }), + ).resolves.toMatchObject({ preparedRuntimeId: NEW_ID }); + expect(fake.replacement?.HostConfig?.GroupAdd).toEqual(["44", "110"]); + expect(fake.replacement?.Config?.Entrypoint).toEqual([ + MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, + ]); + const replacementCommand = fake.replacement?.Config?.Cmd; + assert(Array.isArray(replacementCommand)); + expect(replacementCommand.filter((entry) => entry === JETSON_DEVICE_GROUP_BOOTSTRAP)).toEqual([ + JETSON_DEVICE_GROUP_BOOTSTRAP, + ]); + expect(fake.replacement?.Config?.Cmd?.slice(-7)).toEqual([ + JETSON_DEVICE_GROUP_BOOTSTRAP, + "--device-group-gids", + "44,110", + "--", + "/opt/openshell/bin/openshell-sandbox", + "--workdir", + "/sandbox", + ]); + expect(fake.events).not.toContain(`stop:${OLD_ID}`); + }); + + it("rejects an invalid Jetson device group before replacement creation (#7610)", async () => { + const fake = fixture({ agent: "openclaw" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request } = authority("openclaw"); + const discovered = await adapter.discoverHeldWorkload({ + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + expectedImage: handle.plan.image, + metadata: handle.plan.metadata, + }); + const snapshot = await adapter.inspectHeldWorkload({ handle, discovered }); + + await expect( + adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions: { + values: { extraGroupGids: ["0"] }, + }, + }), + ).rejects.toThrow("invalid or excessive supplementary group IDs"); + expect(fake.events).not.toContain("create:replacement"); + }); + it("accepts equivalent Engine API and Docker CLI create metadata before cutover", async () => { const fake = fixture({ dockerCliSerializationDefaults: true }); const adapter = createDockerManagedBootstrapAdapter(fake.deps); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 8c281feb18c..eac2497136f 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -23,9 +23,14 @@ import { import { buildDockerGpuCloneRunArgs, dockerContainerName, + MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, normalizeDockerUlimitName, shouldOmitOpenShellOciImageUser, } from "../docker-gpu-patch-clone"; +import { + normalizeJetsonDeviceGroupGids, + resolveJetsonDeviceGroupBootstrap, +} from "../docker-gpu-jetson-groups"; import { DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, DOCKER_GPU_PATCH_TIMEOUT_MS, @@ -126,7 +131,7 @@ const DOCKER_DRIVER_ID = "docker"; const MAX_RECOVERY_FAILURE_DETAIL_BYTES = 8 * 1024; const OPENSHELL_DRIVER_IDLE_COMMAND = "sleep infinity"; -export const MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE = "/usr/local/bin/nemoclaw-managed-bootstrap"; +export { MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE }; function boundedRecoveryFailureDetail(error: unknown): string { const raw = (error instanceof Error ? error.message : String(error)).replaceAll("\0", "�"); @@ -715,13 +720,8 @@ function replacementPlan(options: ManagedBootstrapReplacementOptions): { device: String(options.values.gpuModeDevice ?? ""), args, }, - extraGroupGids: exactStringArray(options.values.extraGroupGids ?? [], "extra group GIDs").map( - (value) => { - if (!/^\d+$/u.test(value)) { - throw new Error(`Managed bootstrap Docker supplementary group '${value}' is invalid.`); - } - return value; - }, + extraGroupGids: normalizeJetsonDeviceGroupGids( + exactStringArray(options.values.extraGroupGids ?? [], "extra group GIDs"), ), requiredUlimits: parseRequiredUlimits(options.values.requiredUlimits), }; @@ -730,7 +730,14 @@ function replacementPlan(options: ManagedBootstrapReplacementOptions): { function replacementCommand( handle: ManagedBootstrapHeldWorkloadHandle, snapshot: ManagedBootstrapObservedSnapshot, + plan: ReturnType, ): readonly string[] { + const supervisorArgv = + resolveJetsonDeviceGroupBootstrap({ + agent: handle.plan.profile.agent, + deviceGroupGids: plan.extraGroupGids, + supervisorArgv: snapshot.supervisorArgv, + })?.supervisorArgv ?? snapshot.supervisorArgv; return Object.freeze([ "--agent", handle.plan.profile.agent, @@ -747,20 +754,20 @@ function replacementCommand( "--request-file", MANAGED_BOOTSTRAP_REQUEST_FILE, "--", - ...snapshot.supervisorArgv, + ...supervisorArgv, ]); } function assertReplacementBoundary( inspect: DockerContainerInspect, handle: ManagedBootstrapHeldWorkloadHandle, - snapshot: ManagedBootstrapObservedSnapshot, + expectedCommand: readonly string[], ): void { const entrypoint = exactStringArray(inspect.Config?.Entrypoint, "replacement entrypoint"); const command = exactStringArray(inspect.Config?.Cmd, "replacement command"); if ( !exactArrayEqual(entrypoint, [MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]) || - !exactArrayEqual(command, replacementCommand(handle, snapshot)) + !exactArrayEqual(command, expectedCommand) ) { throw new Error("Managed bootstrap Docker replacement process boundary changed."); } @@ -3453,7 +3460,7 @@ export function createDockerManagedBootstrapAdapter( detail: `preparation requires rollback or commit from durable phase ${existingJournal.phase}`, }); } - const trampolineCommand = replacementCommand(handle, snapshot); + const trampolineCommand = replacementCommand(handle, snapshot, plan); const omitOciImageUser = shouldOmitOpenShellOciImageUser( parsed.inspect, handle.intendedWorkloadArgv, @@ -3520,7 +3527,7 @@ export function createDockerManagedBootstrapAdapter( "Managed bootstrap Docker replacement requires one bounded intended workload argv.", ); } - assertReplacementBoundary(createdInspect, handle, snapshot); + assertReplacementBoundary(createdInspect, handle, trampolineCommand); const expectedActivatedSpecHash = assertReplacementMatchesIntent( snapshot.specCanonicalJson, createdInspect, @@ -3786,7 +3793,12 @@ export function createDockerManagedBootstrapAdapter( (commandDetail(started) || "state did not reach running"), ); } - assertReplacementBoundary(running, handle, snapshot); + const expectedReplacementCommand = exactStringArray( + parseDockerManagedBootstrapLaunchSpec(prepared.expectedActivatedSpecCanonicalJson).inspect + .Config?.Cmd, + "expected replacement command", + ); + assertReplacementBoundary(running, handle, expectedReplacementCommand); const preservedOriginal = inspectExact(snapshot.runtimeId, deps); assertTransactionOriginal(journal, preservedOriginal); if (dockerContainerName(preservedOriginal) !== journal.backupName) { @@ -3853,7 +3865,12 @@ export function createDockerManagedBootstrapAdapter( if (beforeImageContentId !== replacement.runtimeImageContentId) { throw new Error("Managed bootstrap Docker replacement image content changed."); } - assertReplacementBoundary(before, handle, snapshot); + const expectedReplacementCommand = exactStringArray( + parseDockerManagedBootstrapLaunchSpec(replacement.replacementSpecCanonicalJson).inspect + .Config?.Cmd, + "expected replacement command", + ); + assertReplacementBoundary(before, handle, expectedReplacementCommand); const supervisorReconnectTimeoutSecs = getDockerGpuSupervisorReconnectTimeoutSecs(timeoutSecs); if ( @@ -3885,7 +3902,7 @@ export function createDockerManagedBootstrapAdapter( if (assertImage(after, replacement.image, deps) !== replacement.runtimeImageContentId) { throw new Error("Managed bootstrap Docker completed image content changed."); } - assertReplacementBoundary(after, handle, snapshot); + assertReplacementBoundary(after, handle, expectedReplacementCommand); const normalized = normalizeDockerManagedBootstrapLaunchSpec(after); if (normalized.hash !== replacement.replacementSpecHash) { throw new Error("Managed bootstrap Docker replacement changed during bootstrap."); diff --git a/src/lib/sandbox/build-context.ts b/src/lib/sandbox/build-context.ts index 83320664de4..397bc11d740 100644 --- a/src/lib/sandbox/build-context.ts +++ b/src/lib/sandbox/build-context.ts @@ -292,6 +292,10 @@ function stageOptimizedSandboxBuildContext( path.join(rootDir, "scripts", "managed-bootstrap-trampoline.sh"), path.join(stagedScriptsDir, "managed-bootstrap-trampoline.sh"), ); + fs.copyFileSync( + path.join(rootDir, "scripts", "jetson-device-group-bootstrap.sh"), + path.join(stagedScriptsDir, "jetson-device-group-bootstrap.sh"), + ); fs.copyFileSync( path.join(rootDir, "scripts", "gateway-control.sh"), path.join(stagedScriptsDir, "gateway-control.sh"), diff --git a/test/e2e/README.md b/test/e2e/README.md index 130330a3e50..d395b14b2f5 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -202,6 +202,8 @@ It runs the Ubuntu 26.04 compatibility contracts and the full Vitest suite in fo The matrix disables `fail-fast`. The first macOS shard has a 60-minute budget for live E2E; the other shards have 30 minutes. The first WSL shard has a 180-minute budget for root-required contracts and live E2E; the other shards have 90 minutes. +The unprivileged WSL suite excludes `test/runtime/sandbox/jetson-device-group-bootstrap.test.ts`; +the Ubuntu 24.04 `CI / Pull Request` CLI coverage shards run that Docker-backed integration contract. On shard 1, the workflow runs focused macOS and WSL live E2E only when the run tests `main` and Docker is available. Otherwise, the workflow records the skip and retains the platform contract evidence. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index b4b14bf688f..5103b0a61ec 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -354,6 +354,8 @@ test/e2e/ Each macOS shard installs the pinned OpenShell formula. Shard 1 has a 60-minute budget for live E2E; the other shards have 30 minutes. WSL shard 1 has a 180-minute budget for root-required contracts and live E2E; the other shards have 90 minutes. + The unprivileged WSL suite excludes `test/runtime/sandbox/jetson-device-group-bootstrap.test.ts`; + the Ubuntu 24.04 `CI / Pull Request` CLI coverage shards run that Docker-backed integration contract. On shard 1, the workflow runs focused macOS and WSL live E2E only when the run tests `main` and Docker is available. Otherwise, those live tests skip and the platform contracts remain as evidence. This conditional result is platform evidence, not `Release qualification`. diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 6689594782a..1a54731656b 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -195,6 +195,10 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [ pattern: /(?:^|\/)scripts\/setup-jetson\.sh$/, testsToRun: runTests("test/install/setup-jetson.test.ts"), }, + { + pattern: /(?:^|\/)scripts\/jetson-device-group-bootstrap\.sh$/, + testsToRun: runTests("test/runtime/sandbox/jetson-device-group-bootstrap.test.ts"), + }, { pattern: /(?:^|\/)tools\/e2e\/contracts\/v1\/jetson-dispatch\.json$/, testsToRun: runTests("test/e2e/support/jetson-dispatch-client.test.ts"), diff --git a/test/repository/vitest-watch-triggers.test.ts b/test/repository/vitest-watch-triggers.test.ts index 705872331e8..3c2d314a25c 100644 --- a/test/repository/vitest-watch-triggers.test.ts +++ b/test/repository/vitest-watch-triggers.test.ts @@ -71,6 +71,7 @@ const OPAQUE_INPUTS = [ "nemoclaw/src/shared/openshell-policy-boundary.cts", "nemoclaw/tsconfig.shared.json", "scripts/setup-jetson.sh", + "scripts/jetson-device-group-bootstrap.sh", "tools/e2e/contracts/v1/jetson-dispatch.json", ".github/workflows/base-image.yaml", ".github/workflows/base-image-platform.yaml", @@ -253,6 +254,9 @@ describe("Vitest opaque-input watch triggers", () => { "test/e2e/support/hosted-inference.test.ts", ]); expect(triggeredBy("scripts/setup-jetson.sh")).toEqual(["test/install/setup-jetson.test.ts"]); + expect(triggeredBy("scripts/jetson-device-group-bootstrap.sh")).toEqual([ + "test/runtime/sandbox/jetson-device-group-bootstrap.test.ts", + ]); expect(triggeredBy("tools/e2e/contracts/v1/jetson-dispatch.json")).toEqual([ "test/e2e/support/jetson-dispatch-client.test.ts", ]); diff --git a/test/runtime/sandbox/jetson-device-group-bootstrap.test.ts b/test/runtime/sandbox/jetson-device-group-bootstrap.test.ts new file mode 100644 index 00000000000..cd912e45a3e --- /dev/null +++ b/test/runtime/sandbox/jetson-device-group-bootstrap.test.ts @@ -0,0 +1,460 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { dockerSpawnSync } from "../../../src/lib/adapters/docker/exec"; + +const REPO_ROOT = path.join(import.meta.dirname, "../../.."); +const BOOTSTRAP_SCRIPT = path.join(REPO_ROOT, "scripts", "jetson-device-group-bootstrap.sh"); +const FIXTURE_BASE_IMAGE = + "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:16b01f6d7e0b843a72331538f3bf690b6112840064e7d8b81e361a61277bebf7"; +const SUPERVISOR = "/opt/openshell/bin/openshell-sandbox"; +const CONTAINER_TIMEOUT_MS = 20_000; + +type BootstrapRunOptions = { + environment?: Record; + groupDatabase?: "regular" | "symlink"; +}; + +type BootstrapRun = { + after: FixtureState; + before: FixtureState; + status: number | null; + stderr: string; + stdout: string; +}; + +type FixtureState = { + groupAddLog: string; + groupMap: string; + memberships: string; + supervisorArgv: Buffer | null; + usermodLog: string; +}; + +const fixtureParent = process.platform === "darwin" ? "/private/tmp" : os.tmpdir(); +const fixtureId = `${String(process.pid)}-${String(Date.now())}`; +const fixtureImage = `nemoclaw-jetson-bootstrap-test:${fixtureId}`; +let containerFixtureRoot = path.join( + fixtureParent, + `nemoclaw-jetson-bootstrap-not-created-${fixtureId}`, +); + +const GROUP_DATABASE_DOCKER_ARGS = { + regular: [], + symlink: ["--tmpfs", "/etc:rw,nosuid,nodev,noexec,size=1m"], +} as const satisfies Record, readonly string[]>; + +function writeExecutable(filePath: string, source: string): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, source, { mode: 0o755 }); +} + +const INITIAL_STATE: FixtureState = { + groupAddLog: "", + groupMap: "44:video\n", + memberships: "1000 44\n", + supervisorArgv: null, + usermodLog: "", +}; + +function createContainerFixture(): string { + const root = fs.mkdtempSync(path.join(fixtureParent, "nemoclaw-jetson-bootstrap-")); + const usrBin = path.join(root, "usr-bin"); + const usrSbin = path.join(root, "usr-sbin"); + const supervisorDir = path.join(root, "supervisor"); + + writeExecutable( + path.join(usrBin, "id"), + `#!/bin/sh +set -eu +if [ "\${1:-}" = "-u" ]; then + printf '%s\n' "\${TEST_ID_UID:-0}" + exit 0 +fi +if [ "\${1:-}" = "sandbox" ]; then + [ "\${TEST_SANDBOX_MISSING:-0}" != "1" ] || exit 1 + printf 'uid=1000(sandbox) gid=1000(sandbox) groups=%s\n' "$(cat /test-state/memberships)" + exit 0 +fi +if [ "\${1:-}" = "-G" ] && [ "\${2:-}" = "sandbox" ]; then + cat /test-state/memberships + exit 0 +fi +exit 2 +`, + ); + writeExecutable( + path.join(usrBin, "getent"), + `#!/bin/sh +set -eu +[ "\${1:-}" = "group" ] || exit 2 +gid="\${2:-}" +if [ "\${TEST_GETENT_MALFORMED_GID:-}" = "$gid" ]; then + printf ':x:999:\n' + exit 0 +fi +record="$(awk -F: -v gid="$gid" '$1 == gid { print; exit }' /test-state/group-map)" +[ -n "$record" ] || exit 2 +name="\${record#*:}" +printf '%s:x:%s:\n' "$name" "$gid" +`, + ); + writeExecutable( + path.join(usrSbin, "groupadd"), + `#!/bin/sh +set -eu +[ "$#" -eq 3 ] && [ "$1" = "--gid" ] +printf '%s\n' "$*" >>/test-state/groupadd.log +printf '%s:%s\n' "$2" "$3" >>/test-state/group-map +`, + ); + writeExecutable( + path.join(usrSbin, "usermod"), + `#!/bin/sh +set -eu +[ "$#" -eq 4 ] && [ "$1" = "--append" ] && [ "$2" = "--groups" ] && [ "$4" = "sandbox" ] +printf '%s\n' "$*" >>/test-state/usermod.log +[ "\${TEST_USERMOD_NOOP:-0}" != "1" ] || exit 0 +gid="$(awk -F: -v name="$3" '$2 == name { print $1; exit }' /test-state/group-map)" +[ -n "$gid" ] +memberships="$(cat /test-state/memberships)" +case " $memberships " in + *" $gid "*) ;; + *) printf '%s %s\n' "$memberships" "$gid" >/test-state/memberships ;; +esac +`, + ); + writeExecutable( + path.join(supervisorDir, "openshell-sandbox"), + `#!/bin/sh +set -eu +printf '%s\\0' "$@" >/test-state/supervisor.argv +printf 'SUPERVISOR_EXECUTED\n' +`, + ); + writeExecutable( + path.join(root, "fixture-runner"), + `#!/bin/bash +set -uo pipefail +printf '1000 44\n' >/test-state/memberships +printf '44:%s\n' "\${TEST_EXISTING_GROUP_NAME:-video}" >/test-state/group-map +case "\${TEST_GROUP_DATABASE:-regular}" in + regular) ;; + symlink) /bin/ln -s /tmp/nemoclaw-missing-group /etc/group ;; + *) exit 2 ;; +esac +set +e +if [ "\${TEST_RETRY_AFTER_MEMBERSHIP_FAILURE:-0}" = "1" ]; then + export TEST_USERMOD_NOOP=1 + /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh "$@" + first_status=$? + unset TEST_USERMOD_NOOP + [ "$first_status" -ne 0 ] || exit 2 +fi +/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh "$@" +status=$? +set -e +printf 'NEMOCLAW_TEST_STATE_BEGIN\n' +for file in groupadd.log group-map memberships supervisor.argv usermod.log; do + printf '%s=' "$file" + if [ -f "/test-state/$file" ]; then + base64 "/test-state/$file" | tr -d '\n' + fi + printf '\n' +done +printf 'NEMOCLAW_TEST_STATE_END\n' +exit "$status" +`, + ); + fs.copyFileSync(BOOTSTRAP_SCRIPT, path.join(root, "jetson-device-group-bootstrap.sh")); + fs.writeFileSync( + path.join(root, "Dockerfile"), + `FROM ${FIXTURE_BASE_IMAGE} +COPY jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh +COPY fixture-runner /test-fixture/run +COPY supervisor/ /opt/openshell/bin/ +COPY usr-bin/id /usr/bin/id +COPY usr-bin/getent /usr/bin/getent +COPY usr-sbin/ /usr/sbin/ +`, + ); + return root; +} + +function parseFixtureState(stdout: string): { state: FixtureState; stdout: string } { + const startMarker = "NEMOCLAW_TEST_STATE_BEGIN\n"; + const endMarker = "NEMOCLAW_TEST_STATE_END\n"; + const start = stdout.indexOf(startMarker); + const end = stdout.indexOf(endMarker, start + startMarker.length); + expect(start, "fixture state start marker is missing").toBeGreaterThanOrEqual(0); + expect(end, "fixture state end marker is missing").toBeGreaterThan(start); + const encoded = new Map( + stdout + .slice(start + startMarker.length, end) + .trimEnd() + .split("\n") + .map((line) => { + const separator = line.indexOf("="); + return [line.slice(0, separator), line.slice(separator + 1)] as const; + }), + ); + const decode = (name: string): Buffer | null => { + const value = encoded.get(name); + return value ? Buffer.from(value, "base64") : null; + }; + return { + state: { + groupAddLog: decode("groupadd.log")?.toString("utf8") ?? "", + groupMap: decode("group-map")?.toString("utf8") ?? "", + memberships: decode("memberships")?.toString("utf8") ?? "", + supervisorArgv: decode("supervisor.argv"), + usermodLog: decode("usermod.log")?.toString("utf8") ?? "", + }, + stdout: stdout.slice(0, start), + }; +} + +function runBootstrap(args: readonly string[], options: BootstrapRunOptions = {}): BootstrapRun { + const groupDatabase = options.groupDatabase ?? "regular"; + const dockerArgs = [ + "run", + "--rm", + "--network", + "none", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--tmpfs", + "/tmp:rw,nosuid,nodev,noexec,size=1m", + "--tmpfs", + "/test-state:rw,nosuid,nodev,noexec,size=1m", + ...GROUP_DATABASE_DOCKER_ARGS[groupDatabase], + ...Object.entries({ + ...options.environment, + TEST_GROUP_DATABASE: groupDatabase, + }).flatMap(([key, value]) => ["--env", `${key}=${value}`]), + "--entrypoint", + "/test-fixture/run", + fixtureImage, + ...args, + ]; + const result = dockerSpawnSync(dockerArgs, { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: CONTAINER_TIMEOUT_MS, + }); + expect(result.error, result.error?.message).toBeUndefined(); + const parsed = parseFixtureState(String(result.stdout)); + const existingGroupName = options.environment?.TEST_EXISTING_GROUP_NAME ?? "video"; + return { + after: parsed.state, + before: { ...INITIAL_STATE, groupMap: `44:${existingGroupName}\n` }, + status: result.status, + stderr: String(result.stderr), + stdout: parsed.stdout, + }; +} + +function expectNoMutation(run: BootstrapRun): void { + expect(run.after).toEqual(run.before); +} + +const dockerProbe = dockerSpawnSync(["info", "--format", "{{.ServerVersion}}"], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 5_000, +}); +const suite = dockerProbe.status === 0 || process.platform === "linux" ? describe : describe.skip; + +suite("Jetson device-group bootstrap", () => { + beforeAll(() => { + expect( + dockerProbe.status, + `Docker is required for the Linux bootstrap security boundary: ${String(dockerProbe.stderr)}`, + ).toBe(0); + containerFixtureRoot = createContainerFixture(); + const build = dockerSpawnSync( + ["build", "--network", "none", "--tag", fixtureImage, containerFixtureRoot], + { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 60_000, + }, + ); + expect(build.error, build.error?.message).toBeUndefined(); + expect(build.status, `${String(build.stderr)}\n${String(build.stdout)}`).toBe(0); + }, 65_000); + + afterAll(() => { + dockerSpawnSync(["image", "rm", "--force", fixtureImage], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 10_000, + }); + fs.rmSync(containerFixtureRoot, { force: true, recursive: true }); + }); + + it("adds existing and new device groups before the fixed supervisor handoff (#7610)", () => { + const run = runBootstrap([ + "--device-group-gids", + "44,110", + "--", + SUPERVISOR, + "--ready", + "value with space", + ]); + + expect(run.status, run.stderr).toBe(0); + expect(run.stdout).toBe("SUPERVISOR_EXECUTED\n"); + expect(run.after.groupAddLog).toBe("--gid 110 nemoclaw_gpu_110\n"); + expect(run.after.usermodLog).toBe( + "--append --groups video sandbox\n--append --groups nemoclaw_gpu_110 sandbox\n", + ); + expect(run.after.memberships.trim().split(/\s+/)).toEqual(["1000", "44", "110"]); + expect(run.after.supervisorArgv).toEqual(Buffer.from("--ready\0value with space\0", "utf8")); + }); + + it.each([ + { + args: ["--device-group-gids", "44,invalid", "--", SUPERVISOR], + error: "device group ID is invalid", + title: "a later invalid group ID", + }, + { + args: ["--device-group-gids", "44,44", "--", SUPERVISOR], + error: "device group ID is duplicated", + title: "a later duplicate group ID", + }, + { + args: ["--device-group-gids", "2147483648", "--", SUPERVISOR], + error: "device group ID is out of range", + title: "an out-of-range group ID", + }, + { + args: [ + "--device-group-gids", + Array.from({ length: 17 }, (_, index) => String(index + 1)).join(","), + "--", + SUPERVISOR, + ], + error: "device group count is invalid", + title: "more than 16 group IDs", + }, + { + args: ["--device-group-gids", "44", "not-a-delimiter", SUPERVISOR], + error: "supervisor delimiter is missing", + title: "an invalid supervisor delimiter", + }, + { + args: ["--device-group-gids", "44", "--", "/tmp/openshell-sandbox"], + error: "OpenShell supervisor entrypoint is invalid", + title: "a different supervisor entrypoint", + }, + ])("rejects $title before account mutation (#7610)", ({ args, error }) => { + const run = runBootstrap(args); + + expect(run.status).toBe(1); + expect(run.stderr).toContain(`Jetson device-group bootstrap: ${error}`); + expectNoMutation(run); + }); + + it("rejects a missing sandbox account before group mutation (#7610)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + environment: { TEST_SANDBOX_MISSING: "1" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("Jetson device-group bootstrap: sandbox user is missing"); + expectNoMutation(run); + }); + + it("rejects a non-root caller before group mutation (#7610)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + environment: { TEST_ID_UID: "1000" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("Jetson device-group bootstrap: must run as root"); + expectNoMutation(run); + }); + + it("rejects a malformed existing group before account mutation (#7610)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + environment: { TEST_GETENT_MALFORMED_GID: "44" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("Jetson device-group bootstrap: device group record is invalid"); + expectNoMutation(run); + }); + + it("rejects an unapproved existing group before account mutation (#7610)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + environment: { TEST_EXISTING_GROUP_NAME: "shadow" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain( + "Jetson device-group bootstrap: existing device group is not approved", + ); + expectNoMutation(run); + }); + + it("preflights every group record before account mutation (#7610)", () => { + const run = runBootstrap(["--device-group-gids", "44,110", "--", SUPERVISOR], { + environment: { TEST_GETENT_MALFORMED_GID: "110" }, + }); + + expect(run.stderr).toContain("Jetson device-group bootstrap: device group record is invalid"); + expectNoMutation(run); + }); + + it("rejects a symlinked group database before account mutation (#7610)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + groupDatabase: "symlink", + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain( + "Jetson device-group bootstrap: container group database is invalid", + ); + expectNoMutation(run); + }); + + it("stops before supervisor handoff when membership verification fails (#7610)", () => { + const run = runBootstrap(["--device-group-gids", "110", "--", SUPERVISOR], { + environment: { TEST_USERMOD_NOOP: "1" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain( + "Jetson device-group bootstrap: sandbox membership verification failed", + ); + expect(run.after.supervisorArgv).toBeNull(); + expect(run.after.memberships).toBe(run.before.memberships); + }); + + it("reuses an exact generated group when retrying a failed membership handoff (#7610)", () => { + const run = runBootstrap(["--device-group-gids", "110", "--", SUPERVISOR], { + environment: { TEST_RETRY_AFTER_MEMBERSHIP_FAILURE: "1" }, + }); + + expect(run.status, run.stderr).toBe(0); + expect(run.stdout).toBe("SUPERVISOR_EXECUTED\n"); + expect(run.after.groupAddLog).toBe("--gid 110 nemoclaw_gpu_110\n"); + expect(run.after.usermodLog).toBe( + "--append --groups nemoclaw_gpu_110 sandbox\n" + + "--append --groups nemoclaw_gpu_110 sandbox\n", + ); + expect(run.after.memberships.trim().split(/\s+/)).toEqual(["1000", "44", "110"]); + expect(run.after.supervisorArgv).toEqual(Buffer.from([0])); + }); +}); diff --git a/test/runtime/sandbox/sandbox-build-context.test.ts b/test/runtime/sandbox/sandbox-build-context.test.ts index 37069c3c7e3..f15f6b1d656 100644 --- a/test/runtime/sandbox/sandbox-build-context.test.ts +++ b/test/runtime/sandbox/sandbox-build-context.test.ts @@ -224,6 +224,7 @@ describe("sandbox build context staging", () => { fs.chmodSync(path.join(sourceRoot, "nemoclaw-blueprint", "model-specific-setup"), 0o700); fs.chmodSync(blueprintManifestDir, 0o700); writeFixture(path.join("scripts", "nemoclaw-start.sh")); + writeFixture(path.join("scripts", "jetson-device-group-bootstrap.sh")); writeFixture(path.join("scripts", "managed-startup-hold.sh")); writeFixture(path.join("scripts", "managed-bootstrap-entrypoint.c")); writeFixture(path.join("scripts", "managed-bootstrap-trampoline.sh"));