fix(coding-agent): harden daemon startup and recovery ownership

packages/coding-agent/.changes/daemon-recovery-hardening.md

@@ -0,0 +1,2 @@
+- Fixed daemon startup and recovery to preserve slow live processes and fail closed after socket lock loss.
+- Recovery never signals a live worker process it cannot verify as its own: a persistently failing live worker parks as failed with its process left running (reclaimed automatically by the next fresh create once its identity is verified or it exits). The one deliberate exception is replacing an authenticated pre-roster worker during adoption. A live worker that stays silent through ten probe rounds (~2.5 minutes) also parks as failed instead of probing forever.

packages/coding-agent/src/cli/daemon-launch.ts

@@ -64,10 +64,19 @@ async function canConnectToDaemon(socketPath: string, timeoutMs: number): Promis
 type DaemonVersionProbe =
 	| { status: "absent" }
 	| { status: "current"; hello: DaemonHello }
-	| { status: "stale"; hello?: DaemonHello };
+	| { status: "stale"; hello: DaemonHello }
+	| { status: "unresponsive" };
+
+function isCurrentDaemonHello(hello: DaemonHello): boolean {
+	return (
+		hello.protocol.version === DAEMON_PROTOCOL_VERSION &&
+		hello.schemaId === DAEMON_SCHEMA_ID &&
+		hello.appVersion === VERSION
+	);
+}
 
 /** Connect to a running daemon and check whether it matches this client's protocol and app version. */
-export async function probeDaemonVersion(socketPath: string): Promise<DaemonVersionProbe> {
+export async function probeDaemonVersion(socketPath: string, helloTimeoutMs = 2000): Promise<DaemonVersionProbe> {
 	let client: DaemonClient | undefined;
 	for (const timeoutMs of [250, 2000]) {
 		const candidate = new DaemonClient(socketPath);
@@ -83,11 +92,8 @@ export async function probeDaemonVersion(socketPath: string): Promise<DaemonVers
 		return { status: "absent" };
 	}
 	try {
-		const hello = await client.waitForHello(2000);
-		const current =
-			hello.protocol.version === DAEMON_PROTOCOL_VERSION &&
-			hello.schemaId === DAEMON_SCHEMA_ID &&
-			hello.appVersion === VERSION;
+		const hello = await client.waitForHello(helloTimeoutMs);
+		const current = isCurrentDaemonHello(hello);
 		if (!current) {
 			logDaemonLaunch(
 				`running daemon on ${socketPath} is stale: daemon v${hello.appVersion}/proto${hello.protocol.version}` +
@@ -100,9 +106,9 @@ export async function probeDaemonVersion(socketPath: string): Promise<DaemonVers
 		}
 		return { status: "stale", hello };
 	} catch {
-		// Connected but no recognizable greeting: assume a stale daemon.
-		logDaemonLaunch(`running daemon on ${socketPath} sent no recognizable hello; treating as stale`);
-		return { status: "stale" };
+		// The supervisor accepts connections before startup and worker adoption finish.
+		logDaemonLaunch(`running daemon on ${socketPath} sent no recognizable hello; waiting for startup`);
+		return { status: "unresponsive" };
 	} finally {
 		client.close();
 	}
@@ -301,50 +307,70 @@ export async function probeRunningDaemonSessions(socketPath: string): Promise<Ru
 
 // Idle-but-loaded sessions reload from disk on the fresh daemon, so only a busy
 // session blocks replacing a stale daemon.
-async function shutdownStaleDaemonIfNotBusy(socketPath: string): Promise<boolean> {
+type StaleDaemonDisposition = "current" | "stopped" | "busy";
+
+async function shutdownStaleDaemonIfNotBusy(socketPath: string): Promise<StaleDaemonDisposition> {
 	const client = new DaemonClient(socketPath);
-	let connected = false;
-	let hasBusySessions = false;
-	let loadedSessionCount = 0;
 	try {
 		await client.connect(1000);
-		connected = true;
-		try {
-			const result = await queryActiveDaemonSessions(client, { includeClientOwned: true });
-			loadedSessionCount = result.sessions.length;
-			hasBusySessions =
-				result.busyClientOwnedSessionCount !== 0 || result.sessions.some((summary) => isSessionBusy(summary));
-		} catch {
-			// Couldn't confirm idleness: treat as busy rather than risk interrupting work.
-			hasBusySessions = true;
-		}
 	} catch {
-		// Couldn't reach it to inspect; don't send a blind shutdown, just verify below.
-	} finally {
 		client.close();
+		return (await waitForDaemonGone(socketPath)) ? "stopped" : "busy";
 	}
 
-	if (!connected) {
-		return waitForDaemonGone(socketPath);
+	let loadedSessionCount = 0;
+	let hasBusySessions = true;
+	try {
+		const result = await queryActiveDaemonSessions(client, { includeClientOwned: true });
+		loadedSessionCount = result.sessions.length;
+		hasBusySessions =
+			result.busyClientOwnedSessionCount !== 0 || result.sessions.some((summary) => isSessionBusy(summary));
+	} catch {
+		// An unresponsive daemon is not safe to replace.
+	}
+
+	const hello = client.hello;
+	if (hello && isCurrentDaemonHello(hello)) {
+		client.close();
+		logDaemonLaunch(`daemon on ${socketPath} finished starting while staleness was being checked; reusing it`);
+		return "current";
 	}
 	if (hasBusySessions) {
+		client.close();
 		logDaemonLaunch(`refusing to replace stale daemon on ${socketPath}: busy session(s) present`);
-		return false;
+		return "busy";
 	}
 	logDaemonLaunch(
 		`replacing stale daemon on ${socketPath} (idle): ${loadedSessionCount} loaded session(s) will reload`,
 	);
-	return shutdownDaemonAndWait(socketPath);
+	return (await shutdownConnectedDaemonAndWait(client, socketPath, 5000, hello)) ? "stopped" : "busy";
 }
 
 async function ensureDaemonRunning(socketPath: string, spawnCwd?: string): Promise<void> {
-	const probe = await probeDaemonVersion(socketPath);
+	const probeStartedAt = Date.now();
+	let probe = await probeDaemonVersion(socketPath);
+	if (probe.status === "unresponsive") {
+		const remainingStartupMs = Math.max(1, DAEMON_STARTUP_TIMEOUT_MS - (Date.now() - probeStartedAt));
+		probe = await probeDaemonVersion(socketPath, remainingStartupMs);
+	}
 	if (probe.status === "current") {
 		return;
 	}
+	if (probe.status === "unresponsive") {
+		throw new Error(
+			`Prime Agent daemon on ${socketPath} accepted connections but did not finish startup within ${DAEMON_STARTUP_TIMEOUT_MS / 1000} seconds. ` +
+				`It was left running to avoid interrupting active work.
+
+Run:
+${formatCurrentCliCommand(["shutdown", "--force"])}
+
+Then retry the original command.`,
+		);
+	}
 	if (probe.status === "stale") {
-		const stopped = await shutdownStaleDaemonIfNotBusy(socketPath);
-		if (!stopped) throw new StaleDaemonError(socketPath, probe.hello);
+		const disposition = await shutdownStaleDaemonIfNotBusy(socketPath);
+		if (disposition === "current") return;
+		if (disposition === "busy") throw new StaleDaemonError(socketPath, probe.hello);
 	}
 
 	const entrypoint = process.argv[1];

packages/coding-agent/src/cli/daemon-update-restart.ts

@@ -324,11 +324,18 @@ function coordinatorRecordPath(registryDir: string, socketPath: string): string
 
 async function withCoordinatorRegistryGuard<T>(registryDir: string, action: () => T | Promise<T>): Promise<T> {
 	mkdirSync(registryDir, { recursive: true, mode: 0o700 });
+	let compromisedError: Error | undefined;
+	const assertGuardHeld = () => {
+		if (compromisedError) throw new Error(`Coordinator registry guard was compromised: ${compromisedError.message}`);
+	};
 	const release = await lockfile.lock(registryDir, {
 		realpath: false,
 		lockfilePath: resolve(registryDir, ".guard"),
 		stale: COORDINATOR_REGISTRY_LOCK_STALE_MS,
 		update: COORDINATOR_REGISTRY_LOCK_UPDATE_MS,
+		onCompromised: (error) => {
+			compromisedError ??= error;
+		},
 		retries: {
 			retries: COORDINATOR_REGISTRY_LOCK_RETRIES,
 			factor: 1,
@@ -337,9 +344,13 @@ async function withCoordinatorRegistryGuard<T>(registryDir: string, action: () =
 		},
 	});
 	try {
-		return await action();
+		assertGuardHeld();
+		const result = await action();
+		assertGuardHeld();
+		return result;
 	} finally {
-		await release();
+		if (compromisedError) await release().catch(() => undefined);
+		else await release();
 	}
 }
 

packages/coding-agent/src/core/auth-storage.ts

@@ -126,11 +126,23 @@ export class FileAuthStorageBackend implements AuthStorageBackend {
 		const maxAttempts = 10;
 		const delayMs = 20;
 		let lastError: unknown;
+		let compromisedError: Error | undefined;
 
 		for (let attempt = 1; attempt <= maxAttempts; attempt++) {
 			try {
-				return lockfile.lockSync(path, { realpath: false });
+				const release = lockfile.lockSync(path, {
+					realpath: false,
+					onCompromised: (error) => {
+						compromisedError ??= error;
+					},
+				});
+				if (compromisedError) {
+					release();
+					throw compromisedError;
+				}
+				return release;
 			} catch (error) {
+				if (compromisedError) throw compromisedError;
 				const code =
 					typeof error === "object" && error !== null && "code" in error
 						? String((error as { code?: unknown }).code)
@@ -211,11 +223,8 @@ export class FileAuthStorageBackend implements AuthStorageBackend {
 			return result;
 		} finally {
 			if (release) {
-				try {
-					await release();
-				} catch {
-					// Ignore unlock errors when lock is compromised.
-				}
+				if (lockCompromised) await release().catch(() => undefined);
+				else await release();
 			}
 		}
 	}

packages/coding-agent/src/core/cron-jobs.ts

@@ -1499,15 +1499,26 @@ function withCronJobsStateLocks<T>(paths: readonly string[], action: () => T): T
 		for (const path of [...new Set(paths)].sort()) {
 			mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
 			let release: (() => void) | undefined;
+			let lockCompromised = false;
 			for (let attempt = 0; attempt < 100; attempt++) {
 				try {
 					release = lockSync(path, {
 						realpath: false,
 						lockfilePath: `${path}.lock`,
 						stale: 30_000,
+						onCompromised: () => {
+							lockCompromised = true;
+						},
 					});
+					if (lockCompromised) {
+						release();
+						throw new Error(`Cron jobs lock compromised: ${path}`);
+					}
 					break;
 				} catch (error) {
+					if (lockCompromised) {
+						throw new Error(`Cron jobs lock compromised: ${path}`);
+					}
 					if ((error as NodeJS.ErrnoException).code !== "ELOCKED" || attempt === 99) {
 						throw error;
 					}

packages/coding-agent/src/core/session-lease.ts

@@ -187,12 +187,16 @@ function isLeaseOwnerAlive(owner: SessionLeaseOwner): boolean {
 
 function withLeaseGuard<T>(directory: string, action: () => T): T {
 	let release: (() => void) | undefined;
+	let guardCompromised = false;
 	for (let attempt = 0; attempt < 100; attempt++) {
 		try {
 			release = lockSync(directory, {
 				realpath: false,
 				lockfilePath: `${directory}.guard`,
 				stale: 5000,
+				onCompromised: () => {
+					guardCompromised = true;
+				},
 			});
 			break;
 		} catch (error) {
@@ -208,10 +212,24 @@ function withLeaseGuard<T>(directory: string, action: () => T): T {
 	if (!release) {
 		throw new Error(`Could not coordinate session lease: ${directory}`);
 	}
+	const assertGuardHeld = () => {
+		if (guardCompromised) throw new Error(`Session lease guard was compromised: ${directory}`);
+	};
 	try {
-		return action();
+		assertGuardHeld();
+		const result = action();
+		assertGuardHeld();
+		return result;
 	} finally {
-		release();
+		if (guardCompromised) {
+			try {
+				release();
+			} catch {
+				// The compromised guard no longer owns a lock that can be safely released.
+			}
+		} else {
+			release();
+		}
 	}
 }
 

packages/coding-agent/src/core/settings-manager.ts

@@ -236,11 +236,23 @@ export class FileSettingsStorage implements SettingsStorage {
 		const maxAttempts = 10;
 		const delayMs = 20;
 		let lastError: unknown;
+		let compromisedError: Error | undefined;
 
 		for (let attempt = 1; attempt <= maxAttempts; attempt++) {
 			try {
-				return lockfile.lockSync(path, { realpath: false });
+				const release = lockfile.lockSync(path, {
+					realpath: false,
+					onCompromised: (error) => {
+						compromisedError ??= error;
+					},
+				});
+				if (compromisedError) {
+					release();
+					throw compromisedError;
+				}
+				return release;
 			} catch (error) {
+				if (compromisedError) throw compromisedError;
 				const code =
 					typeof error === "object" && error !== null && "code" in error
 						? String((error as { code?: unknown }).code)

packages/coding-agent/src/modes/daemon/daemon-catalog-entry.ts

@@ -0,0 +1,7 @@
+import { runDaemonCatalogProcess } from "./daemon-catalog-process.js";
+
+runDaemonCatalogProcess().catch((error: unknown) => {
+	const message = error instanceof Error ? error.message : String(error);
+	process.stderr.write(`Prime Agent daemon catalog failed: ${message}\n`);
+	process.exit(1);
+});