Skip to content

Commit 269cff6

Browse files
authored
fix(server): prevent Linux workspace launch identity timeouts (NeuralNomadsAI#625)
## Summary - prevent Linux workspace startup from failing when process identity discovery exceeds its one-second command deadline - capture the launched process and its existing process-group members without spawning helper commands for every `/proc` entry - preserve the leader-exit cleanup guarantees introduced by NeuralNomadsAI#602 - read immutable Linux process start ticks correctly and tolerate multiline task names ## Root cause Workspace startup captures an immutable process identity before accepting the OpenCode runtime. The Linux implementation scanned every process and launched several `cat`, `cut`, `sed`, `basename`, and `dirname` helpers per entry. On the affected Mint host, that synchronous shell command exceeded its one-second timeout. Identity capture then failed closed and stopped the newly launched OpenCode process; cleanup retried the same expensive probes and produced the repeated `spawnSync sh ETIMEDOUT` errors. The identity parser also used `$20`, which POSIX shells interpret as `$2` followed by `0`, rather than the twentieth positional field. This did not cause the startup timeout but weakened immutable process matching and is corrected here. ## Safety - launch discovery still retains every already-started member of the observed process group - guarded cleanup remains identity- and launch-token-based; no unverified PID fallback is introduced - WSL, macOS, and Windows paths retain their existing platform-specific probes ## Validation - server TypeScript typecheck - focused process identity and runtime suite: 23 passed, 1 Linux-only test skipped on Windows - real WSL `/proc` probe: correct start identity, 2 group members, 4 ms - broader workspace suite: 79 passed, 3 platform skips - `git diff --check` The broader workspace run still has the unrelated existing Windows fixture failure in `git-worktrees.test.ts` (`undefined` instead of `main`). Closes NeuralNomadsAI#624
1 parent 70c9548 commit 269cff6

4 files changed

Lines changed: 74 additions & 14 deletions

File tree

‎packages/server/src/workspaces/process-identity.test.ts‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import assert from "node:assert/strict"
2-
import type { SpawnSyncReturns } from "node:child_process"
2+
import { spawn as spawnChild, spawnSync, type SpawnSyncReturns } from "node:child_process"
3+
import { once } from "node:events"
4+
import { readFileSync } from "node:fs"
35
import { describe, it } from "node:test"
46

57
import {
@@ -28,6 +30,33 @@ describe("process identity probes", () => {
2830
assert.deepEqual(probe.ok && probe.processes.get(42), identity())
2931
})
3032

33+
it("queries the requested Linux launch group without per-process subprocesses", () => {
34+
const call = {} as Call
35+
const probe = probePosixProcesses(spawn("42|1|42|123456|boot-a|123456\n", call), 25, "linux", { pids: [42], groupId: 42 })
36+
assert.deepEqual(call.args.slice(-1), ["42"])
37+
assert.match(call.script, /expected_group=\$stat_group/)
38+
assert.doesNotMatch(call.script, /\b(?:cat|cut|sed|basename|dirname)\b/)
39+
assert.deepEqual(probe.ok && probe.processes.get(42), identity())
40+
})
41+
42+
it("captures real Linux start ticks and launch-group members within the deadline", { skip: process.platform !== "linux" }, async () => {
43+
const child = spawnChild("sh", ["-c", "sleep 5"], { stdio: "ignore" })
44+
await once(child, "spawn")
45+
try {
46+
const stat = readFileSync(`/proc/${process.pid}/stat`, "utf8")
47+
const expectedStart = stat.slice(stat.lastIndexOf(") ") + 2).split(" ")[19]
48+
const probe = probePosixProcesses(spawnSync, 1_000, "linux", { pids: [process.pid], groupId: process.pid })
49+
assert.equal(probe.ok && probe.processes.get(process.pid)?.startTime, expectedStart)
50+
assert.equal(probe.ok && probe.processes.has(child.pid!), true)
51+
} finally {
52+
if (child.exitCode === null && child.signalCode === null) {
53+
const exited = once(child, "exit")
54+
child.kill()
55+
await exited
56+
}
57+
}
58+
})
59+
3160
it("uses one delimiter-safe process-table query on portable POSIX", () => {
3261
const call = {} as Call
3362
const command = "/opt/opencode 'pipe|value'\t\"quoted\" café"

‎packages/server/src/workspaces/process-identity.ts‎

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -36,20 +36,36 @@ export type TokenSignalResult = { ok: boolean; signalSent: boolean; targets: Pro
3636
export const LAUNCH_CLEANUP_TOKEN_ENV = "CODENOMAD_LAUNCH_CLEANUP_TOKEN"
3737

3838
type SpawnCommand = typeof spawnSync
39+
const SHELL_DOLLAR = "$"
3940

4041
const LINUX_IDENTITY_FUNCTIONS = String.raw`
41-
boot=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null) || exit 20
42+
IFS= read -r boot 2>/dev/null < /proc/sys/kernel/random/boot_id || exit 20
4243
read_stat() {
43-
line=$(cat "/proc/$1/stat" 2>/dev/null) || return 1
44-
stat_pid=$(printf '%s\n' "$line" | cut -d' ' -f1); rest=$(printf '%s\n' "$line" | sed 's/^.*) //'); set -- $rest
45-
stat_ppid=$2; stat_group=$3; stat_start=$20
44+
line=
45+
while IFS= read -r chunk || test -n "$chunk"; do line=$line$chunk; done 2>/dev/null < "/proc/$1/stat"
46+
test -n "$line" || return 1
47+
stat_pid=$1; rest=${SHELL_DOLLAR}{line##*) }; set -- $rest
48+
test "$#" -ge 20 || return 1
49+
stat_ppid=$2; stat_group=$3; shift 19; stat_start=$1
50+
}
51+
emit_linux() {
52+
test -n "$1" && printf '%s|' "$1"
53+
printf '%s|%s|%s|%s|%s|%s\n' "$stat_pid" "$stat_ppid" "$stat_group" "$stat_start" "$boot" "$stat_start"
4654
}
47-
emit_linux() { printf '%s|%s|%s|%s|%s|%s|%s\n' "$1" "$stat_pid" "$stat_ppid" "$stat_group" "$stat_start" "$boot" "$stat_start"; }
4855
`
4956

5057
const LINUX_SNAPSHOT_SCRIPT = String.raw`${LINUX_IDENTITY_FUNCTIONS}
5158
for stat in /proc/[0-9]*/stat; do
52-
pid=$(basename "$(dirname "$stat")"); read_stat "$pid" && emit_linux "" | cut -c2-
59+
directory=${SHELL_DOLLAR}{stat%/stat}; pid=${SHELL_DOLLAR}{directory##*/}; read_stat "$pid" && emit_linux ""
60+
done
61+
exit 0
62+
`
63+
64+
const LINUX_LAUNCH_GROUP_SNAPSHOT_SCRIPT = String.raw`${LINUX_IDENTITY_FUNCTIONS}
65+
leader_pid=$1; read_stat "$leader_pid" || exit 22; expected_group=$stat_group; emit_linux ""
66+
for stat in /proc/[0-9]*/stat; do
67+
directory=${SHELL_DOLLAR}{stat%/stat}; pid=${SHELL_DOLLAR}{directory##*/}; test "$pid" = "$leader_pid" && continue
68+
read_stat "$pid" && test "$stat_group" = "$expected_group" && emit_linux ""
5369
done
5470
exit 0
5571
`
@@ -60,7 +76,7 @@ shift 5; matched=0; cutoff=; signal_sent=0
6076
if read_stat "$leader_pid" && test "$boot" = "$leader_boot" && test "$stat_start" = "$leader_start" && test "$stat_group" = "$expected_group"; then
6177
matched=1
6278
for stat in /proc/[0-9]*/stat; do
63-
candidate=$(basename "$(dirname "$stat")"); read_stat "$candidate" && test "$stat_group" = "$expected_group" && emit_linux CODENOMAD_TARGET
79+
directory=${SHELL_DOLLAR}{stat%/stat}; candidate=${SHELL_DOLLAR}{directory##*/}; read_stat "$candidate" && test "$stat_group" = "$expected_group" && emit_linux CODENOMAD_TARGET
6480
done
6581
if kill "-$requested_signal" -- "-$expected_group" 2>/dev/null; then
6682
signal_sent=1
@@ -111,7 +127,7 @@ pass=0
111127
while test "$pass" -lt "$passes"; do
112128
pass=$((pass + 1))
113129
for environ in /proc/[0-9]*/environ; do
114-
pid=$(basename "$(dirname "$environ")")
130+
directory=${SHELL_DOLLAR}{environ%/environ}; pid=${SHELL_DOLLAR}{directory##*/}
115131
if matches_token "$pid" && read_stat "$pid"; then
116132
test -n "$requested_signal" && prefix=CODENOMAD_TARGET || prefix=CODENOMAD_PROCESS
117133
emit_linux "$prefix"
@@ -416,10 +432,16 @@ export function descendantsOf(processes: Map<number, ProcessIdentity>, rootPid:
416432

417433
export function probePosixProcesses(spawnCommand: SpawnCommand, timeoutMs: number,
418434
platform: NodeJS.Platform = process.platform, filter?: PosixProcessFilter): ProcessSnapshot {
419-
if (platform === "linux") return querySnapshot(
420-
() => runLinuxScript(spawnCommand, LINUX_SNAPSHOT_SCRIPT, [], timeoutMs, "codenomad-posix-identity"),
421-
(output) => parseDelimitedSnapshot(output, true),
422-
)
435+
if (platform === "linux") {
436+
const pids = filter?.pids?.filter((pid) => Number.isInteger(pid) && pid > 0).map(String) ?? []
437+
const launchGroupProbe = pids.length === 1 && filter?.groupId === Number(pids[0])
438+
return querySnapshot(
439+
() => runLinuxScript(spawnCommand, launchGroupProbe ? LINUX_LAUNCH_GROUP_SNAPSHOT_SCRIPT : LINUX_SNAPSHOT_SCRIPT,
440+
launchGroupProbe ? pids : [], timeoutMs, "codenomad-posix-identity"),
441+
(output) => parseDelimitedSnapshot(output, true),
442+
{ allowEmpty: Boolean(filter) },
443+
)
444+
}
423445
// POSIX has no portable pidfd/start ticks; collect one coherent table instead of probing every PID.
424446
return querySnapshot(
425447
() => spawnCommand("ps", ["-axo", "pid=,ppid=,pgid=,lstart=,comm="], {

‎packages/server/src/workspaces/runtime.test.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,15 @@ async function harness(options: WorkspaceRuntimeOptions & { binary?: string; out
8383
return { runtime, child, timers, calls, launch, abort }
8484
}
8585
describe("workspace runtime lifecycle contracts", () => {
86+
it("captures the Linux launch group with one bounded shell command", async () => {
87+
let launchCall: Call | undefined
88+
await harness({ spawnSync: ((command: string, args: readonly string[]) => {
89+
launchCall ??= { command, args: [...args] }
90+
return result(posix([[4242, 1, 4242, "100"]]))
91+
}) as unknown as Command })
92+
assert.deepEqual(launchCall?.args.slice(-1), ["4242"])
93+
})
94+
8695
it("cancels before spawn and while waiting for a port without losing retryable cleanup", async () => {
8796
let spawned = false
8897
const runtime = new WorkspaceRuntime(new EventBus(), pino({ level: "silent" }), {

‎packages/server/src/workspaces/runtime.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ export class WorkspaceRuntime {
276276
this.spawnCommand,
277277
this.stopCommandTimeoutMs,
278278
this.platform,
279-
this.platform === "linux" ? undefined : { pids: [child.pid], groupId: child.pid },
279+
{ pids: [child.pid], groupId: child.pid },
280280
)
281281
: { ok: false as const, error: "spawned child did not expose a PID" }
282282
const launchLeader = launchSnapshot.ok && child.pid ? launchSnapshot.processes.get(child.pid) : undefined

0 commit comments

Comments
 (0)