diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 032ea08b84..50289e8fef 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Added `doctor` reporting of stray IPython kernels and stale kernel temp dirs; `doctor --fix` kills the orphaned kernels and removes the stale dirs, and `doctor --json` gains a `kernels` section. - Fixed large IPython variables repeatedly slowing later turns by excluding them from persistent snapshots and removing them when context is compacted. - Fixed daemon socket paths being used verbatim in identity derivations: on supported platforms, `--daemon-socket` spellings differing only by duplicate or trailing slashes now normalize to one canonical path, so worker-descriptor namespaces, daemon log files, and persisted descriptors agree. - Added a `thinking` option to `rlm.run` for spawning subagents with an explicit reasoning level; invalid levels for the resolved child model fail spawn. diff --git a/packages/coding-agent/src/cli/command-registry.ts b/packages/coding-agent/src/cli/command-registry.ts index c11a4de382..40d8a9f346 100644 --- a/packages/coding-agent/src/cli/command-registry.ts +++ b/packages/coding-agent/src/cli/command-registry.ts @@ -83,7 +83,10 @@ export const COMMAND_SPECS: readonly CommandSpec[] = [ path: ["doctor"], usage: "doctor [--fix] [--json]", summary: "Inspect and safely clean up background services", - options: ["--fix Remove stale sockets and stop idle orphaned services", "--json Print JSON"], + options: [ + "--fix Remove stale sockets, stop idle orphaned services, kill stray IPython kernels, and remove stale kernel temp dirs", + "--json Print JSON", + ], }, { path: ["shutdown"], diff --git a/packages/coding-agent/src/cli/daemon-ps.ts b/packages/coding-agent/src/cli/daemon-ps.ts index ea1d16de53..2ecf250f31 100644 --- a/packages/coding-agent/src/cli/daemon-ps.ts +++ b/packages/coding-agent/src/cli/daemon-ps.ts @@ -17,6 +17,13 @@ import type { DaemonWorkerDescriptor } from "../modes/daemon/daemon-worker-proto import { signalProcessGroupOrProcess } from "../utils/child-process.js"; import { formatDaemonListTable } from "./daemon-ps-format.js"; import { promptYesNo } from "./daemon-stop-confirm.js"; +import { + formatKernelFixResult, + formatKernelReport, + type KernelFindings, + type KernelFixResult, + kernelJsonSummary, +} from "./doctor-kernel-reap.js"; /** * `daemon ps` discovers every prime-agent daemon on the machine, not just the @@ -396,17 +403,21 @@ export function sortDaemons(infos: DaemonInfo[]): DaemonInfo[] { }); } -export async function runPs(json: boolean): Promise { +export async function runPs(json: boolean, kernels?: KernelFindings): Promise { const daemons = await discoverDaemons(); if (json) { - console.log(JSON.stringify(daemons, null, 2)); + console.log(JSON.stringify(kernels ? { daemons, kernels: kernelJsonSummary(kernels) } : daemons, null, 2)); return; } if (daemons.length === 0) { console.log("No background services found."); - return; + } else { + console.log(formatDaemonListTable(daemons)); + } + const kernelReport = kernels ? formatKernelReport(kernels) : undefined; + if (kernelReport) { + console.log(kernelReport); } - console.log(formatDaemonListTable(daemons)); } export type ReapAction = @@ -1067,7 +1078,7 @@ async function stopTrackedProcess( return !isProcessAlive(pid); } -export async function runReap(json: boolean, force: boolean): Promise { +export async function runReap(json: boolean, force: boolean, kernelFix?: KernelFixResult): Promise { const daemons = await discoverDaemons(); const reaped: Array<{ socketPath: string; action: string }> = []; const skipped: Array<{ socketPath: string; reason: string }> = []; @@ -1113,10 +1124,11 @@ export async function runReap(json: boolean, force: boolean): Promise { } if (json) { - console.log(JSON.stringify({ reaped, skipped }, null, 2)); + console.log(JSON.stringify(kernelFix ? { reaped, skipped, kernels: kernelFix } : { reaped, skipped }, null, 2)); return; } - if (reaped.length === 0 && skipped.length === 0) { + const kernelReport = kernelFix ? formatKernelFixResult(kernelFix) : undefined; + if (reaped.length === 0 && skipped.length === 0 && kernelReport === undefined) { console.log("No background services found."); return; } @@ -1126,6 +1138,9 @@ export async function runReap(json: boolean, force: boolean): Promise { for (const entry of skipped) { console.log(chalk.dim(`kept ${entry.socketPath}: ${entry.reason}`)); } + if (kernelReport) { + console.log(kernelReport); + } } type ReapOutcome = { reaped: string } | { skipped: string }; diff --git a/packages/coding-agent/src/cli/doctor-kernel-reap.ts b/packages/coding-agent/src/cli/doctor-kernel-reap.ts new file mode 100644 index 0000000000..028345f2a4 --- /dev/null +++ b/packages/coding-agent/src/cli/doctor-kernel-reap.ts @@ -0,0 +1,451 @@ +import { spawnSync } from "node:child_process"; +import { readdirSync, rmSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; +import chalk from "chalk"; + +export interface KernelProcess { + pid: number; + ppid: number; + connectionPath: string; + tempDir: string; +} + +export interface KernelFindings { + strays: KernelProcess[]; + owned: KernelProcess[]; + staleTempDirs: string[]; + leakedTestDaemons: number[]; + /** ps rows referencing a forkserver socket: the template or its forked kernels. */ + forkedKernelsPresent: number; + /** Report-only: forkserver-backed kernels orphaned to init (see ENG-5310). */ + orphanedForkedKernels: number[]; + psUnavailable: boolean; +} + +export interface KernelFixResult { + killedStrays: number[]; + removedTempDirs: number; + skipped: Array<{ pid?: number; path?: string; reason: string }>; + leakedTestDaemons: number[]; + forkedKernelsPresent: number; + orphanedForkedKernels: number[]; + psUnavailable: boolean; +} + +const KERNEL_TEMP_DIR_PREFIX = "prime-agent-kernel-"; +// Forked kernels keep the template's argv (fork-server.ts), so this socket-dir marker is their only ps signature. +const FORK_SERVER_MARKER = "prime-agent-forkserver-"; +// Anchored full-argv match so wrappers merely mentioning ipykernel_launcher are never treated as kernels. +const KERNEL_COMMAND_PATTERN = /^(\S+) -m ipykernel_launcher -f (\S+\/connection\.json)$/; +const PYTHON_BINARY_PATTERN = /^python[\d.]*$/; +// Below this age a missing live reference may just be a kernel mid-startup. +const STALE_TEMP_DIR_AGE_MS = 60 * 60 * 1000; +// Huge process tables overflow spawnSync's 1MiB default, which would silently disable all checks. +const PS_MAX_BUFFER = 10 * 1024 * 1024; + +function parseKernelCommand(command: string): { connectionPath: string; tempDir: string } | undefined { + const match = command.match(KERNEL_COMMAND_PATTERN); + if (!match || !PYTHON_BINARY_PATTERN.test(basename(match[1]!))) { + return undefined; + } + const connectionPath = match[2]!; + const tempDir = dirname(connectionPath); + return basename(tempDir).startsWith(KERNEL_TEMP_DIR_PREFIX) ? { connectionPath, tempDir } : undefined; +} + +/** Parse `ps -axo pid=,ppid=,args=` output into prime-agent IPython kernel processes. */ +export function parseKernelProcesses(stdout: string): KernelProcess[] { + const kernels: KernelProcess[] = []; + for (const line of stdout.split("\n")) { + const fields = line.trim().match(/^(\d+)\s+(\d+)\s+(.+)$/); + if (!fields) { + continue; + } + const command = parseKernelCommand(fields[3]!.trim()); + if (!command) { + continue; + } + kernels.push({ + pid: Number.parseInt(fields[1]!, 10), + ppid: Number.parseInt(fields[2]!, 10), + connectionPath: command.connectionPath, + tempDir: command.tempDir, + }); + } + return kernels; +} + +/** Report-only: leaked eng-4600 test daemon fixtures orphaned to init. */ +export function parseLeakedTestDaemons(stdout: string): number[] { + const pids: number[] = []; + for (const line of stdout.split("\n")) { + const fields = line.trim().match(/^(\d+)\s+(\d+)\s+(.+)$/); + if (!fields) { + continue; + } + if (Number.parseInt(fields[2]!, 10) === 1 && fields[3]!.includes("eng-4600-supervisor-fixture")) { + pids.push(Number.parseInt(fields[1]!, 10)); + } + } + return pids; +} + +/** ps rows referencing a forkserver socket dir: the template process or kernels forked from it. */ +export function parseForkedKernelRows(stdout: string): Array<{ pid: number; ppid: number }> { + const rows: Array<{ pid: number; ppid: number }> = []; + for (const line of stdout.split("\n")) { + const fields = line.trim().match(/^(\d+)\s+(\d+)\s+(.+)$/); + if (fields?.[3]?.includes(FORK_SERVER_MARKER)) { + rows.push({ pid: Number.parseInt(fields[1]!, 10), ppid: Number.parseInt(fields[2]!, 10) }); + } + } + return rows; +} + +/** A kernel is stray only when orphaned to init; any live parent means owned. */ +export function classifyStrayKernels(kernels: KernelProcess[]): { strays: KernelProcess[]; owned: KernelProcess[] } { + const strays: KernelProcess[] = []; + const owned: KernelProcess[] = []; + for (const kernel of kernels) { + (kernel.ppid === 1 ? strays : owned).push(kernel); + } + return { strays, owned }; +} + +export function classifyStaleTempDirs( + dirs: ReadonlyArray<{ path: string; mtimeMs: number }>, + psStdout: string, + nowMs: number, +): string[] { + // Loose substring match: any live command mentioning the connection path protects the dir, kernel or not. + return dirs + .filter( + (dir) => !psStdout.includes(join(dir.path, "connection.json")) && nowMs - dir.mtimeMs > STALE_TEMP_DIR_AGE_MS, + ) + .map((dir) => dir.path); +} + +export interface KernelScanProbes { + /** Full `ps -axo pid=,ppid=,args=` output, or undefined when ps failed. */ + runPs: () => string | undefined; + listTempDirs: () => Array<{ path: string; mtimeMs: number }>; + now: () => number; +} + +const defaultScanProbes: KernelScanProbes = { + runPs: () => { + const ps = spawnSync("ps", ["-axo", "pid=,ppid=,args="], { encoding: "utf8", maxBuffer: PS_MAX_BUFFER }); + return !ps.error && ps.status === 0 && typeof ps.stdout === "string" ? ps.stdout : undefined; + }, + listTempDirs: scanKernelTempDirs, + now: Date.now, +}; + +export async function scanKernelFindings(probes: KernelScanProbes = defaultScanProbes): Promise { + const empty: KernelFindings = { + strays: [], + owned: [], + staleTempDirs: [], + leakedTestDaemons: [], + forkedKernelsPresent: 0, + orphanedForkedKernels: [], + psUnavailable: false, + }; + if (process.platform === "win32") { + return empty; + } + const stdout = probes.runPs(); + if (stdout === undefined) { + // Without a process list liveness can't be established; fail closed and report nothing. + return { ...empty, psUnavailable: true }; + } + const { strays, owned } = classifyStrayKernels(parseKernelProcesses(stdout)); + const forkedRows = parseForkedKernelRows(stdout); + return { + strays, + owned, + // Forked kernels lack their connection path in argv, so their live dirs would look unreferenced; fail closed. + staleTempDirs: forkedRows.length > 0 ? [] : classifyStaleTempDirs(probes.listTempDirs(), stdout, probes.now()), + leakedTestDaemons: parseLeakedTestDaemons(stdout), + forkedKernelsPresent: forkedRows.length, + orphanedForkedKernels: forkedRows.filter((row) => row.ppid === 1).map((row) => row.pid), + psUnavailable: false, + }; +} + +function scanKernelTempDirs(): Array<{ path: string; mtimeMs: number }> { + const dirs: Array<{ path: string; mtimeMs: number }> = []; + let entries: string[]; + try { + entries = readdirSync(tmpdir()); + } catch { + return []; + } + for (const entry of entries) { + if (!entry.startsWith(KERNEL_TEMP_DIR_PREFIX)) { + continue; + } + const path = join(tmpdir(), entry); + try { + const stats = statSync(path); + if (stats.isDirectory()) { + dirs.push({ path, mtimeMs: stats.mtimeMs }); + } + } catch { + // Entry vanished between readdir and stat; ignore. + } + } + return dirs; +} + +export interface KernelReapHooks { + recheckStray: (kernel: KernelProcess) => boolean; + /** Resolves true only when the process is confirmed gone. */ + killProcess: (kernel: KernelProcess) => Promise; + removeDir: (path: string) => void; + /** Only direct prime-agent-kernel-* children of this root may be removed. */ + tempRoot: string; +} + +/** Pure recheck: the ps line must still show the same init-parented kernel command. */ +export function confirmsStrayKernel(psLine: string, kernel: KernelProcess): boolean { + const fields = psLine.trim().match(/^(\d+)\s+(.+)$/); + if (!fields) { + return false; + } + const command = parseKernelCommand(fields[2]!.trim()); + return Number.parseInt(fields[1]!, 10) === 1 && command?.connectionPath === kernel.connectionPath; +} + +export type PidIdentity = "stray-kernel" | "gone" | "other"; + +// Guards pid reuse: distinguishes the original orphaned kernel from an exited pid or a new owner. +function checkPidIdentity(kernel: KernelProcess): PidIdentity { + const ps = spawnSync("ps", ["-o", "ppid=,args=", "-p", String(kernel.pid)], { + encoding: "utf8", + maxBuffer: PS_MAX_BUFFER, + }); + if (ps.error || typeof ps.stdout !== "string") { + // ps itself failed; fail closed as if another process owned the pid. + return "other"; + } + if (ps.status !== 0 || ps.stdout.trim() === "") { + return "gone"; + } + return confirmsStrayKernel(ps.stdout, kernel) ? "stray-kernel" : "other"; +} + +export interface KernelKillProbes { + kill: (pid: number, signal: NodeJS.Signals) => void; + pidIdentity: (kernel: KernelProcess) => PidIdentity; + waitForExit: (pid: number, timeoutMs: number) => Promise; +} + +const defaultKillProbes: KernelKillProbes = { + kill: (pid, signal) => process.kill(pid, signal), + pidIdentity: checkPidIdentity, + waitForExit, +}; + +export async function forceKillKernel( + kernel: KernelProcess, + probes: KernelKillProbes = defaultKillProbes, +): Promise { + try { + probes.kill(kernel.pid, "SIGTERM"); + } catch (error) { + // ESRCH means already gone; anything else (e.g. EPERM) is a failure. + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + if (await probes.waitForExit(kernel.pid, 1000)) { + return true; + } + // The pid may have been reused since SIGTERM; only escalate if it still shows the same kernel. + const identity = probes.pidIdentity(kernel); + if (identity === "gone") { + return true; + } + if (identity === "other") { + return false; + } + try { + probes.kill(kernel.pid, "SIGKILL"); + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + return probes.waitForExit(kernel.pid, 1000); +} + +async function waitForExit(pid: number, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (isProcessAlive(pid)) { + if (Date.now() >= deadline) { + return false; + } + await delay(50); + } + return true; +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +const defaultReapHooks: KernelReapHooks = { + recheckStray: (kernel) => checkPidIdentity(kernel) === "stray-kernel", + killProcess: forceKillKernel, + removeDir: (path) => rmSync(path, { recursive: true, force: true }), + tempRoot: tmpdir(), +}; + +export async function reapKernelFindings( + findings: KernelFindings, + hooks: KernelReapHooks = defaultReapHooks, +): Promise { + const killedStrays: number[] = []; + const skipped: Array<{ pid?: number; path?: string; reason: string }> = []; + let removedTempDirs = 0; + const removeDir = (path: string): void => { + // Never delete outside the bounded sweep, even for paths parsed from system-wide ps rows. + if (dirname(path) !== hooks.tempRoot || !basename(path).startsWith(KERNEL_TEMP_DIR_PREFIX)) { + skipped.push({ path, reason: "outside current temp root; not removing" }); + return; + } + try { + hooks.removeDir(path); + removedTempDirs++; + } catch { + skipped.push({ path, reason: "could not remove temp dir" }); + } + }; + for (const kernel of findings.strays) { + if (!hooks.recheckStray(kernel)) { + skipped.push({ pid: kernel.pid, reason: "no longer an orphaned kernel; not killing" }); + continue; + } + if (!(await hooks.killProcess(kernel))) { + // Keep the temp dir: the kernel may still be alive and using it. + skipped.push({ pid: kernel.pid, reason: "could not confirm kernel exit; keeping its temp dir" }); + continue; + } + killedStrays.push(kernel.pid); + removeDir(kernel.tempDir); + } + for (const path of findings.staleTempDirs) { + removeDir(path); + } + return { + killedStrays, + removedTempDirs, + skipped, + leakedTestDaemons: findings.leakedTestDaemons, + forkedKernelsPresent: findings.forkedKernelsPresent, + orphanedForkedKernels: findings.orphanedForkedKernels, + psUnavailable: findings.psUnavailable, + }; +} + +export function formatKernelReport(findings: KernelFindings): string | undefined { + const lines: string[] = []; + if (findings.psUnavailable) { + lines.push(chalk.dim("! could not scan processes (ps failed); skipping kernel checks")); + } + if (findings.strays.length > 0) { + const pids = findings.strays.map((kernel) => kernel.pid).join(", "); + lines.push( + chalk.yellow( + `! ${findings.strays.length} stray IPython kernel(s) (orphaned): pids ${pids} — run "prime-agent doctor --fix"`, + ), + ); + } + if (findings.staleTempDirs.length > 0) { + lines.push( + chalk.yellow( + `! ${findings.staleTempDirs.length} stale kernel temp dir(s) in $TMPDIR — run "prime-agent doctor --fix"`, + ), + ); + } + if (findings.forkedKernelsPresent > 0) { + lines.push(formatForkedKernelNotes(findings)); + } + if (findings.leakedTestDaemons.length > 0) { + lines.push(formatLeakedTestDaemonWarning(findings.leakedTestDaemons)); + } + return lines.length > 0 ? lines.join("\n") : undefined; +} + +function formatForkedKernelNotes(findings: KernelFindings | KernelFixResult): string { + const lines = [ + chalk.dim( + `! ${findings.forkedKernelsPresent} forkserver-backed kernel process(es) running; skipping stale temp dir checks`, + ), + ]; + if (findings.orphanedForkedKernels.length > 0) { + lines.push( + chalk.dim( + `! ${findings.orphanedForkedKernels.length} orphaned forkserver-backed kernel(s) (pids ${findings.orphanedForkedKernels.join(", ")}) — not auto-fixed (see ENG-5310)`, + ), + ); + } + return lines.join("\n"); +} + +function formatLeakedTestDaemonWarning(pids: number[]): string { + return chalk.dim( + `! ${pids.length} leaked test daemon(s) (eng-4600 fixture, pids ${pids.join(", ")}) — not auto-fixed`, + ); +} + +export function formatKernelFixResult(result: KernelFixResult): string | undefined { + const lines: string[] = []; + if (result.psUnavailable) { + lines.push(chalk.dim("! could not scan processes (ps failed); skipping kernel checks")); + } + if (result.killedStrays.length > 0 || result.removedTempDirs > 0 || result.skipped.length > 0) { + lines.push( + chalk.green( + `reaped kernels: killed ${result.killedStrays.length} stray kernel(s)` + + (result.killedStrays.length > 0 ? ` (pids ${result.killedStrays.join(", ")})` : "") + + `, removed ${result.removedTempDirs} temp dir(s)`, + ), + ); + } + for (const skip of result.skipped) { + lines.push(chalk.dim(`kept ${skip.pid !== undefined ? `pid ${skip.pid}` : skip.path}: ${skip.reason}`)); + } + if (result.forkedKernelsPresent > 0) { + lines.push(formatForkedKernelNotes(result)); + } + if (result.leakedTestDaemons.length > 0) { + lines.push(formatLeakedTestDaemonWarning(result.leakedTestDaemons)); + } + return lines.length > 0 ? lines.join("\n") : undefined; +} + +export function kernelJsonSummary(findings: KernelFindings): { + strays: number; + staleTempDirs: number; + leakedTestDaemons: number; + forkedKernelsPresent: number; + orphanedForkedKernels: number; + psUnavailable: boolean; +} { + return { + strays: findings.strays.length, + staleTempDirs: findings.staleTempDirs.length, + leakedTestDaemons: findings.leakedTestDaemons.length, + forkedKernelsPresent: findings.forkedKernelsPresent, + orphanedForkedKernels: findings.orphanedForkedKernels.length, + psUnavailable: findings.psUnavailable, + }; +} diff --git a/packages/coding-agent/src/cli/public-command.ts b/packages/coding-agent/src/cli/public-command.ts index 020498e1d8..4bd900f7a1 100644 --- a/packages/coding-agent/src/cli/public-command.ts +++ b/packages/coding-agent/src/cli/public-command.ts @@ -15,6 +15,7 @@ import { import { handleDaemonCommand } from "./daemon-command.js"; import { runPs, runReap, runShutdownAll } from "./daemon-ps.js"; import { DAEMON_UPDATE_RESTART_COORDINATOR_FLAG } from "./daemon-update-restart.js"; +import { reapKernelFindings, scanKernelFindings } from "./doctor-kernel-reap.js"; export interface PublicCommandResult { handled: boolean; @@ -250,10 +251,12 @@ async function runStatus(args: string[]): Promise { async function runDoctor(args: string[]): Promise { const options = parseBooleanOptions(args, new Set(["--fix", "--json"]), "doctor"); if (!options) return HANDLED; + const kernels = await scanKernelFindings(); if (options.has("--fix")) { - await runReap(options.has("--json"), false); + const result = await reapKernelFindings(kernels); + await runReap(options.has("--json"), false, result); } else { - await runPs(options.has("--json")); + await runPs(options.has("--json"), kernels); } return HANDLED; } diff --git a/packages/coding-agent/test/doctor-kernel-reap.test.ts b/packages/coding-agent/test/doctor-kernel-reap.test.ts new file mode 100644 index 0000000000..0a695bd2fc --- /dev/null +++ b/packages/coding-agent/test/doctor-kernel-reap.test.ts @@ -0,0 +1,396 @@ +import { describe, expect, it } from "vitest"; +import { + classifyStaleTempDirs, + classifyStrayKernels, + confirmsStrayKernel, + forceKillKernel, + type KernelFindings, + type KernelProcess, + type KernelReapHooks, + parseForkedKernelRows, + parseKernelProcesses, + parseLeakedTestDaemons, + reapKernelFindings, + scanKernelFindings, +} from "../src/cli/doctor-kernel-reap.js"; + +const HOUR_MS = 60 * 60 * 1000; + +function kernel(pid: number, ppid: number, tempDir: string): KernelProcess { + return { pid, ppid, connectionPath: `${tempDir}/connection.json`, tempDir }; +} + +function findings(partial: Partial): KernelFindings { + return { + strays: [], + owned: [], + staleTempDirs: [], + leakedTestDaemons: [], + forkedKernelsPresent: 0, + orphanedForkedKernels: [], + psUnavailable: false, + ...partial, + }; +} + +const FORK_SERVER_ROW = (pid: number, ppid: number): string => + ` ${pid} ${ppid} /home/u/.prime/agent/kernel-venv/bin/python -c import gc,os,sys /tmp/prime-agent-forkserver-ab12/control.sock`; + +describe("parseKernelProcesses", () => { + const stdout = [ + " 111 1 /home/u/.venv/bin/python -m ipykernel_launcher -f /tmp/prime-agent-kernel-abc123/connection.json", + " 222 500 /usr/bin/python3 -m ipykernel_launcher -f /tmp/jupyter-runtime/connection.json", + " 333 1 /usr/bin/python3 scripts/train.py", + " 444 1 prime-agent --mode daemon", + "", + ].join("\n"); + + it("extracts only prime-agent kernel processes with pid, ppid, and temp dir", () => { + expect(parseKernelProcesses(stdout)).toEqual([ + { + pid: 111, + ppid: 1, + connectionPath: "/tmp/prime-agent-kernel-abc123/connection.json", + tempDir: "/tmp/prime-agent-kernel-abc123", + }, + ]); + }); + + it("rejects lookalike commands that only mention ipykernel_launcher", () => { + const lookalikes = [ + " 111 1 node wrapper -m ipykernel_launcher -f /tmp/prime-agent-kernel-x/connection.json", + " 222 1 /usr/bin/python -m ipykernel_launcher -f /tmp/prime-agent-kernel-x/connection.json --extra", + " 333 1 sh -c 'python -m ipykernel_launcher -f /tmp/prime-agent-kernel-x/connection.json'", + " 444 1 /opt/pythonish -m ipykernel_launcher -f /tmp/prime-agent-kernel-x/connection.json", + "", + ].join("\n"); + expect(parseKernelProcesses(lookalikes)).toEqual([]); + }); +}); + +describe("parseLeakedTestDaemons", () => { + it("reports eng-4600 fixture processes only when orphaned to init", () => { + const stdout = [ + " 555 1 node test/eng-4600-supervisor-fixture.ts --daemon", + " 666 777 node test/eng-4600-supervisor-fixture.ts --daemon", + " 888 1 node unrelated.js", + "", + ].join("\n"); + expect(parseLeakedTestDaemons(stdout)).toEqual([555]); + }); +}); + +describe("parseForkedKernelRows", () => { + it("matches only rows referencing a forkserver socket dir", () => { + const stdout = [ + FORK_SERVER_ROW(900, 1), + " 111 1 /home/u/.venv/bin/python -m ipykernel_launcher -f /tmp/prime-agent-kernel-abc/connection.json", + " 333 1 node unrelated.js", + "", + ].join("\n"); + expect(parseForkedKernelRows(stdout)).toEqual([{ pid: 900, ppid: 1 }]); + }); +}); + +describe("classifyStrayKernels", () => { + it("treats only exactly init-parented kernels as stray", () => { + const orphan = kernel(111, 1, "/tmp/prime-agent-kernel-a"); + const attached = kernel(222, 4321, "/tmp/prime-agent-kernel-b"); + const kernelParent = kernel(333, 0, "/tmp/prime-agent-kernel-c"); + expect(classifyStrayKernels([orphan, attached, kernelParent])).toEqual({ + strays: [orphan], + owned: [attached, kernelParent], + }); + }); +}); + +describe("classifyStaleTempDirs", () => { + const now = 10 * HOUR_MS; + const psStdout = + " 111 1 /usr/bin/python3 -m ipykernel_launcher -f /tmp/prime-agent-kernel-live/connection.json\n"; + + it("marks old unreferenced dirs stale", () => { + expect( + classifyStaleTempDirs([{ path: "/tmp/prime-agent-kernel-old", mtimeMs: now - 2 * HOUR_MS }], psStdout, now), + ).toEqual(["/tmp/prime-agent-kernel-old"]); + }); + + it("keeps dirs referenced by a live kernel", () => { + expect( + classifyStaleTempDirs([{ path: "/tmp/prime-agent-kernel-live", mtimeMs: now - 2 * HOUR_MS }], psStdout, now), + ).toEqual([]); + }); + + it("keeps old dirs referenced by any live command, even non-killable lookalikes", () => { + const lookalike = + " 111 1 node wrapper -m ipykernel_launcher -f /tmp/prime-agent-kernel-wrapped/connection.json\n"; + expect( + classifyStaleTempDirs( + [{ path: "/tmp/prime-agent-kernel-wrapped", mtimeMs: now - 2 * HOUR_MS }], + lookalike, + now, + ), + ).toEqual([]); + }); + + it("keeps fresh dirs even when unreferenced", () => { + expect( + classifyStaleTempDirs([{ path: "/tmp/prime-agent-kernel-new", mtimeMs: now - HOUR_MS / 2 }], psStdout, now), + ).toEqual([]); + }); +}); + +describe("scanKernelFindings", () => { + it("fails closed when ps is unavailable: reports nothing fixable", async () => { + const result = await scanKernelFindings({ + runPs: () => undefined, + listTempDirs: () => [{ path: "/tmp/prime-agent-kernel-old", mtimeMs: 0 }], + now: () => 10 * HOUR_MS, + }); + expect(result).toEqual(findings({ psUnavailable: true })); + }); + + it("classifies stale dirs against the live kernel set when ps succeeds", async () => { + const result = await scanKernelFindings({ + runPs: () => + " 111 1 /usr/bin/python3 -m ipykernel_launcher -f /tmp/prime-agent-kernel-live/connection.json\n", + listTempDirs: () => [ + { path: "/tmp/prime-agent-kernel-live", mtimeMs: 0 }, + { path: "/tmp/prime-agent-kernel-old", mtimeMs: 0 }, + ], + now: () => 10 * HOUR_MS, + }); + expect(result.strays).toEqual([kernel(111, 1, "/tmp/prime-agent-kernel-live")]); + expect(result.staleTempDirs).toEqual(["/tmp/prime-agent-kernel-old"]); + expect(result.psUnavailable).toBe(false); + }); + + it("skips stale temp dir classification entirely while forkserver-backed kernels run", async () => { + const withForkServer = await scanKernelFindings({ + runPs: () => `${FORK_SERVER_ROW(900, 800)}\n`, + listTempDirs: () => [{ path: "/tmp/prime-agent-kernel-old", mtimeMs: 0 }], + now: () => 10 * HOUR_MS, + }); + expect(withForkServer.staleTempDirs).toEqual([]); + expect(withForkServer.forkedKernelsPresent).toBe(1); + const control = await scanKernelFindings({ + runPs: () => " 900 800 /usr/bin/python3 unrelated.py\n", + listTempDirs: () => [{ path: "/tmp/prime-agent-kernel-old", mtimeMs: 0 }], + now: () => 10 * HOUR_MS, + }); + expect(control.staleTempDirs).toEqual(["/tmp/prime-agent-kernel-old"]); + expect(control.forkedKernelsPresent).toBe(0); + }); + + it("reports orphaned forkserver-backed kernels without ever classifying them as strays", async () => { + const result = await scanKernelFindings({ + runPs: () => `${FORK_SERVER_ROW(901, 1)}\n${FORK_SERVER_ROW(902, 800)}\n`, + listTempDirs: () => [], + now: () => 10 * HOUR_MS, + }); + expect(result.strays).toEqual([]); + expect(result.forkedKernelsPresent).toBe(2); + expect(result.orphanedForkedKernels).toEqual([901]); + }); + + it("protects an old dir referenced by a live command the kill classifier rejects", async () => { + const result = await scanKernelFindings({ + runPs: () => + " 111 1 node wrapper -m ipykernel_launcher -f /tmp/prime-agent-kernel-wrapped/connection.json\n", + listTempDirs: () => [{ path: "/tmp/prime-agent-kernel-wrapped", mtimeMs: 0 }], + now: () => 10 * HOUR_MS, + }); + expect(result.strays).toEqual([]); + expect(result.staleTempDirs).toEqual([]); + }); +}); + +describe("confirmsStrayKernel", () => { + const stray = kernel(111, 1, "/tmp/prime-agent-kernel-a"); + + it("confirms the same init-parented kernel command", () => { + expect( + confirmsStrayKernel( + " 1 /usr/bin/python3 -m ipykernel_launcher -f /tmp/prime-agent-kernel-a/connection.json", + stray, + ), + ).toBe(true); + }); + + it("rejects lookalikes, reparented kernels, and other connection files", () => { + for (const line of [ + " 1 node wrapper -m ipykernel_launcher -f /tmp/prime-agent-kernel-a/connection.json", + " 4321 /usr/bin/python3 -m ipykernel_launcher -f /tmp/prime-agent-kernel-a/connection.json", + " 1 /usr/bin/python3 -m ipykernel_launcher -f /tmp/prime-agent-kernel-b/connection.json", + "", + ]) { + expect(confirmsStrayKernel(line, stray)).toBe(false); + } + }); +}); + +describe("reapKernelFindings", () => { + function fakeHooks( + recheck: (kernel: KernelProcess) => boolean, + killOutcome = true, + ): { hooks: KernelReapHooks; killed: number[]; removed: string[] } { + const killed: number[] = []; + const removed: string[] = []; + return { + hooks: { + recheckStray: recheck, + killProcess: async (target) => { + killed.push(target.pid); + return killOutcome; + }, + removeDir: (path) => { + removed.push(path); + }, + tempRoot: "/tmp", + }, + killed, + removed, + }; + } + + it("kills confirmed strays and removes their dirs plus stale dirs", async () => { + const { hooks, killed, removed } = fakeHooks(() => true); + const result = await reapKernelFindings( + findings({ + strays: [kernel(111, 1, "/tmp/prime-agent-kernel-a")], + staleTempDirs: ["/tmp/prime-agent-kernel-stale"], + }), + hooks, + ); + expect(killed).toEqual([111]); + expect(removed).toEqual(["/tmp/prime-agent-kernel-a", "/tmp/prime-agent-kernel-stale"]); + expect(result).toEqual({ + killedStrays: [111], + removedTempDirs: 2, + skipped: [], + leakedTestDaemons: [], + forkedKernelsPresent: 0, + orphanedForkedKernels: [], + psUnavailable: false, + }); + }); + + it("declines to kill a kernel whose recheck shows a live parent", async () => { + const { hooks, killed, removed } = fakeHooks(() => false); + const result = await reapKernelFindings( + findings({ strays: [kernel(111, 1, "/tmp/prime-agent-kernel-a")] }), + hooks, + ); + expect(killed).toEqual([]); + expect(removed).toEqual([]); + expect(result.killedStrays).toEqual([]); + expect(result.skipped).toEqual([{ pid: 111, reason: "no longer an orphaned kernel; not killing" }]); + }); + + it("keeps the temp dir and reports a skip when the kill is not confirmed", async () => { + const { hooks, killed, removed } = fakeHooks(() => true, false); + const result = await reapKernelFindings( + findings({ strays: [kernel(111, 1, "/tmp/prime-agent-kernel-a")] }), + hooks, + ); + expect(killed).toEqual([111]); + expect(removed).toEqual([]); + expect(result.killedStrays).toEqual([]); + expect(result.skipped).toEqual([{ pid: 111, reason: "could not confirm kernel exit; keeping its temp dir" }]); + }); + + it("kills a stray in a foreign temp root but leaves its directory untouched", async () => { + const { hooks, killed, removed } = fakeHooks(() => true); + const result = await reapKernelFindings( + findings({ strays: [kernel(111, 1, "/var/other-tmp/prime-agent-kernel-a")] }), + hooks, + ); + expect(killed).toEqual([111]); + expect(removed).toEqual([]); + expect(result.killedStrays).toEqual([111]); + expect(result.skipped).toEqual([ + { path: "/var/other-tmp/prime-agent-kernel-a", reason: "outside current temp root; not removing" }, + ]); + }); + + it("keeps the dir and records a skip when the pre-SIGKILL identity recheck shows a different command", async () => { + const stray = kernel(111, 1, "/tmp/prime-agent-kernel-a"); + const signals: NodeJS.Signals[] = []; + const removed: string[] = []; + const result = await reapKernelFindings(findings({ strays: [stray] }), { + recheckStray: () => true, + killProcess: (target) => + forceKillKernel(target, { + kill: (_pid, signal) => { + signals.push(signal); + }, + pidIdentity: () => "other", + waitForExit: async () => false, + }), + removeDir: (path) => { + removed.push(path); + }, + tempRoot: "/tmp", + }); + expect(signals).toEqual(["SIGTERM"]); + expect(removed).toEqual([]); + expect(result.killedStrays).toEqual([]); + expect(result.skipped).toEqual([{ pid: 111, reason: "could not confirm kernel exit; keeping its temp dir" }]); + }); + + it("never touches owned kernels and passes leaked test daemons through for reporting", async () => { + const { hooks, killed, removed } = fakeHooks(() => true); + const result = await reapKernelFindings( + findings({ owned: [kernel(222, 4321, "/tmp/prime-agent-kernel-b")], leakedTestDaemons: [555] }), + hooks, + ); + expect(killed).toEqual([]); + expect(removed).toEqual([]); + expect(result.leakedTestDaemons).toEqual([555]); + }); +}); + +describe("forceKillKernel", () => { + const stray = kernel(111, 1, "/tmp/prime-agent-kernel-a"); + + it("does not send SIGKILL when the pid now belongs to a different process", async () => { + const signals: NodeJS.Signals[] = []; + const confirmed = await forceKillKernel(stray, { + kill: (_pid, signal) => { + signals.push(signal); + }, + // Survives SIGTERM, but by SIGKILL time the pid shows a different command. + pidIdentity: () => "other", + waitForExit: async () => false, + }); + expect(signals).toEqual(["SIGTERM"]); + expect(confirmed).toBe(false); + }); + + it("treats a pid that vanished after SIGTERM as a confirmed exit", async () => { + const signals: NodeJS.Signals[] = []; + const confirmed = await forceKillKernel(stray, { + kill: (_pid, signal) => { + signals.push(signal); + }, + pidIdentity: () => "gone", + waitForExit: async () => false, + }); + expect(signals).toEqual(["SIGTERM"]); + expect(confirmed).toBe(true); + }); + + it("escalates to SIGKILL only when the pid still shows the same orphaned kernel", async () => { + const signals: NodeJS.Signals[] = []; + let waits = 0; + const confirmed = await forceKillKernel(stray, { + kill: (_pid, signal) => { + signals.push(signal); + }, + pidIdentity: () => "stray-kernel", + waitForExit: async () => ++waits > 1, + }); + expect(signals).toEqual(["SIGTERM", "SIGKILL"]); + expect(confirmed).toBe(true); + }); +});