fix(test): treat exited zombie panes as stopped during cleanup (#107)

An exited Linux pane can keep its `/proc` entry and start tick while
waiting to be reaped. The tmux test harness treated that zombie as a
running process after pidfd had already signaled exit, so rollback could
report a surviving pane or attempt escalation against an exited process.

Reuse the existing proc-state parser to exclude zombies while retaining
exact generation checks. Add a real fork/waitid regression that holds a
child unreaped and covers running, stale-generation, zombie, and reaped
states. The regression fails before the fix and passes after it;
production tmux control is unchanged.

Validation: `npm run verify` passed on Node 24.18.0: 1,448 server tests,
49 real tmux/PTY tests, 521 client tests, and 29 Rust tests, with zero
failures or skips. Type checks, lint, identity, audit at the existing
threshold, and the production build passed. Local release metadata
validation also passed.

Related investigation: #105 failed its rollback assertion with an
aggregate cleanup error in run 33912030795. That intermittent CI failure
did not reproduce in 350 local startup/rollback iterations; this PR
proves and fixes the zombie boundary independently. Release-grade CUA
was not run.

Author: devswha

server/modules/providers/tests/support/tmux-owned-server.ts

@@ -6,6 +6,8 @@ import { promisify } from 'node:util';
 
 import { spawn as spawnPty, type IPty } from 'node-pty';
 
+import { parseProcStatState } from '../../services/process-start-time.service.js';
+
 const execFileAsync = promisify(execFile);
 const EXIT_TIMEOUT_MS = 5_000;
 
@@ -33,10 +35,12 @@ async function readOwnedProcess(pid: number): Promise<OwnedPaneProcess> {
   return { pid, processGroupId, startedAtTicks: processStartTicks(stat) };
 }
 
-async function stillOwned(process: OwnedPaneProcess): Promise<boolean> {
+export async function isOwnedPaneRunning(process: OwnedPaneProcess): Promise<boolean> {
   try {
     const stat = await import('node:fs/promises').then(({ readFile }) => readFile(`/proc/${process.pid}/stat`, 'utf8'));
-    return processStartTicks(stat) === process.startedAtTicks;
+    // pidfd signals exit before the parent necessarily reaps /proc. A zombie
+    // retains its start tick but cannot run or receive an escalation signal.
+    return processStartTicks(stat) === process.startedAtTicks && parseProcStatState(stat) !== 'Z';
   } catch (error) {
     if (error instanceof Error && 'code' in error && (error.code === 'ENOENT' || error.code === 'ESRCH')) return false;
     throw error;
@@ -154,7 +158,7 @@ export async function startOwnedTmuxServer(
       try { await run(['kill-server']); } catch { pty.kill('SIGTERM'); }
       const escalation = setTimeout(() => {
         for (const owned of panes) {
-          void stillOwned(owned).then((ownedNow) => {
+          void isOwnedPaneRunning(owned).then((ownedNow) => {
             if (!ownedNow) return;
             try { process.kill(-owned.processGroupId, 'SIGKILL'); } catch (error) {
               if (!(error instanceof Error && 'code' in error && error.code === 'ESRCH')) throw error;
@@ -166,7 +170,7 @@ export async function startOwnedTmuxServer(
       await Promise.all([ptyExit.exited, ...paneExits.map(({ exited }) => exited)]);
       clearTimeout(escalation);
       for (const owned of panes) {
-        if (await stillOwned(owned)) throw new OwnedTmuxError(`Owned pane PID ${owned.pid} survived tmux shutdown.`);
+        if (await isOwnedPaneRunning(owned)) throw new OwnedTmuxError(`Owned pane PID ${owned.pid} survived tmux shutdown.`);
       }
       try { process.kill(pty.pid, 0); } catch (error) {
         if (error instanceof Error && 'code' in error && error.code === 'ESRCH') return;

server/modules/providers/tests/tmux-fleet-lifecycle.e2e.test.ts

@@ -1,14 +1,16 @@
 import assert from 'node:assert/strict';
 import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from 'node:child_process';
-import { stat } from 'node:fs/promises';
+import { readFile, stat } from 'node:fs/promises';
 import path from 'node:path';
 import test from 'node:test';
 import { pathToFileURL } from 'node:url';
 
 import { startFleetServers, stopFleetProcesses, type FleetProcess } from '../../../../scripts/cua/fleet-process-lifecycle.js';
+import { parseProcStatStartTicks, parseProcStatState } from '../services/process-start-time.service.js';
 
 import { createTmuxFleetE2EHarness } from './support/tmux-fleet-harness.js';
 import { createTmuxFleetNode } from './support/tmux-fleet-node.js';
+import { isOwnedPaneRunning } from './support/tmux-owned-server.js';
 import type { TmuxFleetNode } from './support/tmux-e2e-types.js';
 
 const tmuxE2ESkip = process.platform === 'win32'
@@ -70,6 +72,48 @@ function startFleetWorker(): ChildProcessWithoutNullStreams {
   });
 }
 
+test('owned pane cleanup distinguishes an exited zombie from a running generation', {
+  skip: process.platform !== 'linux', timeout: 10_000, concurrency: false,
+}, async (t) => {
+  // Keep an exited child unreaped until the assertion, just as pidfd readiness
+  // can precede its parent's waitpid during concurrent tmux shutdown.
+  const source = [
+    'import os,sys',
+    'reader,writer=os.pipe()',
+    'pid=os.fork()',
+    'if pid==0:',
+    ' os.close(writer); os.read(reader,1); os._exit(0)',
+    'os.close(reader)',
+    "print('LIVE='+str(pid),flush=True)",
+    'try:',
+    ' sys.stdin.readline(); os.close(writer)',
+    ' os.waitid(os.P_PID,pid,os.WEXITED|os.WNOWAIT)',
+    " print('ZOMBIE=',flush=True)",
+    ' sys.stdin.readline()',
+    'finally: os.waitpid(pid,0)',
+  ].join('\n');
+  const child = spawn('python3', ['-c', source], { stdio: ['pipe', 'pipe', 'pipe'] });
+  const exited = waitForExit(child);
+  t.after(async () => { child.stdin.end(); await exited; });
+  const pid = Number(await waitForLine(child, 'LIVE='));
+  const initialStat = await readFile(`/proc/${pid}/stat`, 'utf8');
+  const ticks = parseProcStatStartTicks(initialStat);
+  assert.ok(ticks !== null);
+  const identity = { pid, processGroupId: pid, startedAtTicks: String(ticks) };
+  assert.equal(await isOwnedPaneRunning(identity), true);
+  assert.equal(await isOwnedPaneRunning({ ...identity, startedAtTicks: String(ticks + 1) }), false);
+
+  const zombie = waitForLine(child, 'ZOMBIE=');
+  child.stdin.write('EXIT\n');
+  await zombie;
+  assert.equal(parseProcStatState(await readFile(`/proc/${pid}/stat`, 'utf8')), 'Z');
+  assert.equal(await isOwnedPaneRunning(identity), false, 'an exited PID must not be escalated or reported as surviving shutdown');
+
+  child.stdin.end();
+  await exited;
+  assert.equal(await isOwnedPaneRunning(identity), false);
+});
+
 test('fleet harness rolls back fulfilled siblings when one concurrent node start fails', {
   skip: tmuxE2ESkip, timeout: 30_000, concurrency: false,
 }, async () => {