diff --git a/docs/assets/codex-session-auto-upgrade/settings.png b/docs/assets/codex-session-auto-upgrade/settings.png new file mode 100644 index 000000000..1691f626b Binary files /dev/null and b/docs/assets/codex-session-auto-upgrade/settings.png differ diff --git a/src/adapters/cli/codex-app.ts b/src/adapters/cli/codex-app.ts index 89896ef16..0569a9f9d 100644 --- a/src/adapters/cli/codex-app.ts +++ b/src/adapters/cli/codex-app.ts @@ -62,7 +62,7 @@ export function createCodexAppAdapter(pathOverride?: string): CliAdapter { return [(cachedCodexBin ??= resolveCommandReal(rawCodexBin))]; }, - buildArgs({ sessionId, resume, resumeSessionId, workingDir, botName, botOpenId, locale, model, reasoningEffort, codexBrowser }) { + buildArgs({ sessionId, resume, resumeSessionId, quietResume, workingDir, botName, botOpenId, locale, model, reasoningEffort, codexBrowser }) { const args = [ runnerArgv0('codex-app-runner', runnerPath()), '--session-id', sessionId, @@ -71,6 +71,7 @@ export function createCodexAppAdapter(pathOverride?: string): CliAdapter { '--codex-bin', (cachedCodexBin ??= resolveCommandReal(rawCodexBin)), ]; if (resume && resumeSessionId) args.push('--thread-id', resumeSessionId); + if (quietResume) args.push('--strict-resume'); pushOpt(args, '--cwd', workingDir); pushOpt(args, '--bot-name', botName); pushOpt(args, '--bot-open-id', botOpenId); diff --git a/src/adapters/cli/codex.ts b/src/adapters/cli/codex.ts index 5385bc74d..41f8cb2df 100644 --- a/src/adapters/cli/codex.ts +++ b/src/adapters/cli/codex.ts @@ -177,7 +177,7 @@ export function createCodexAdapter(pathOverride?: string): CliAdapter { authPaths: ['~/.codex'], get resolvedBin(): string { return (cachedBin ??= resolveCommand(rawBin)); }, - buildArgs({ sessionId, resume, resumeSessionId, forkSession, workingDir, model, reasoningEffort, disableCliBypass, bypassHookTrust, readIsolation, remoteWsUrl, remoteThreadId, shellSubprocessEnv }) { + buildArgs({ sessionId, resume, resumeSessionId, quietResume, forkSession, workingDir, model, reasoningEffort, disableCliBypass, bypassHookTrust, readIsolation, remoteWsUrl, remoteThreadId, shellSubprocessEnv }) { // Hybrid RPC input mode: attach this TUI to the botmux-owned app-server // thread. User input is delivered out-of-band via JSON-RPC (turn/start, // see codex-rpc-engine + worker), so the pane is a pure viewer — no paste @@ -188,7 +188,8 @@ export function createCodexAdapter(pathOverride?: string): CliAdapter { // enter to continue" dialog would block the resume forever and freeze the // Web terminal. Disable the check at the PROCESS level (never the user's // global config). The bounded startup-dialog watcher is only a fail-safe. - return ['--remote', remoteWsUrl, 'resume', '--no-alt-screen', '-c', 'check_for_update_on_startup=false', remoteThreadId]; + return ['--remote', remoteWsUrl, 'resume', '--no-alt-screen', '-c', 'check_for_update_on_startup=false', + ...(quietResume ? ['-c', 'tui.auto_recap=false'] : []), remoteThreadId]; } // Read isolation for Codex is enforced by the worker's Seatbelt wrapper, // NOT by codex's own profile (codex 0.137 can't express a read blocklist). @@ -275,7 +276,8 @@ export function createCodexAdapter(pathOverride?: string): CliAdapter { // privilege-escalation guard on fork. Falls back to plain `resume` when we // somehow lack a source id (nothing to fork from). const codexArgs = codexSessionId - ? [forkSession ? 'fork' : 'resume', ...baseArgs, codexSessionId] + ? [forkSession ? 'fork' : 'resume', ...baseArgs, + ...(quietResume && !forkSession ? ['-c', 'tui.auto_recap=false'] : []), codexSessionId] : freshArgs; return codexArgs; }, diff --git a/src/adapters/cli/types.ts b/src/adapters/cli/types.ts index 8f1fe409b..7575b680f 100644 --- a/src/adapters/cli/types.ts +++ b/src/adapters/cli/types.ts @@ -131,6 +131,9 @@ export interface CliAdapter { workingDir?: string; /** CLI-native session id used for resume when it differs from botmux's session id. */ resumeSessionId?: string; + /** Maintenance resume with no new input: suppress automatic recap/inference + * and require the original thread where the adapter supports strict resume. */ + quietResume?: boolean; /** When true, resume the `resumeSessionId` transcript but write forward into a * NEW CLI-native session id instead of the resumed one, leaving the source * transcript untouched — the native "fork/branch a session" primitive diff --git a/src/codex-app-runner.ts b/src/codex-app-runner.ts index cbd248410..3f06e65d8 100644 --- a/src/codex-app-runner.ts +++ b/src/codex-app-runner.ts @@ -52,6 +52,7 @@ interface Args { controlSocketPath?: string; controlLocatorPath?: string; threadId?: string; + strictResume?: boolean; botName?: string; botOpenId?: string; locale?: string; @@ -258,6 +259,7 @@ function parseArgs(argv: string[]): Args { else if (key === '--codex-bin' && val !== undefined) { out.codexBin = val; i++; } else if (key === '--cwd' && val !== undefined) { out.cwd = val; i++; } else if (key === '--thread-id' && val !== undefined) { out.threadId = val; i++; } + else if (key === '--strict-resume') out.strictResume = true; else if (key === '--bot-name' && val !== undefined) { out.botName = val; i++; } else if (key === '--bot-open-id' && val !== undefined) { out.botOpenId = val; i++; } else if (key === '--locale' && val !== undefined) { out.locale = val; i++; } @@ -267,6 +269,7 @@ function parseArgs(argv: string[]): Args { else if (key === '--browser-plugin-root' && val !== undefined) { out.browserPluginRoot = val; i++; } } if (!out.sessionId) throw new Error('--session-id is required'); + if (out.strictResume && !out.threadId) throw new Error('--strict-resume requires --thread-id'); if (!controlBootstrapPath) throw new Error(`${CODEX_APP_CONTROL_BOOTSTRAP_ENV} is required`); const control = consumeCodexAppControlBootstrap(controlBootstrapPath, out.sessionId); out.controlGeneration = control.generation; @@ -1504,6 +1507,9 @@ async function ensureThread(startupDeadlineAtMs?: number): Promise { ...(browserBroker ? { dynamicTools: [CODEX_BROWSER_DYNAMIC_TOOL] } : {}), }, { timeoutMs: startupRequestTimeout(startupDeadlineAtMs, 'thread/resume') }); const resumedThreadId = String(resumed.thread.id); + if (args.strictResume && resumedThreadId !== threadId) { + throw new Error(`Strict resume expected thread ${threadId}, received ${resumedThreadId}`); + } threadId = resumedThreadId; threadReady = true; emitMarker('thread', { threadId: resumedThreadId }); @@ -1511,9 +1517,10 @@ async function ensureThread(startupDeadlineAtMs?: number): Promise { } catch (err: any) { // A transport error or timeout is an ambiguous acceptance boundary. It // must never fork history by silently creating a fresh thread. Only an - // explicit app-server "missing thread" rejection permits fallback. + // explicit app-server "missing thread" rejection permits normal fallback; + // maintenance resumes must preserve the original thread in every case. if (isActiveWriterConflict(err)) throw new CodexAppActiveWriterError(threadId, err); - if (!isExplicitMissingThread(err)) throw err; + if (args.strictResume || !isExplicitMissingThread(err)) throw err; writeLine(`[codex-app] resume failed, starting a fresh thread: ${err?.message ?? err}`); threadId = undefined; threadReady = false; diff --git a/src/config.ts b/src/config.ts index b7139348a..4f8c0d5c3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -353,6 +353,9 @@ export const config = { // ON. A per-bot codexRpcInput:true still force-enables; the dashboard toggle // sets this global explicitly. get codexRpcInputDefault(): boolean { return readGlobalConfig().dashboard?.codexRpcInput === true; }, + // Default ON; read live so a Dashboard change gates the next session upgrade + // without restarting daemons or changing the current turn. + get autoUpgradeCodexSessions(): boolean { return readGlobalConfig().dashboard?.autoUpgradeCodexSessions !== false; }, // Live getter (like codexRpcInputDefault): re-reads the experimental global // toggle that gates the "no visible output" anti-resend guidance in the botmux // routing hints, so a Settings change takes effect on the next session without diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index 9eff3e58c..44a2ab41d 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -12148,6 +12148,19 @@ function setupWorkerHandlers( break; } + case 'cli_runtime_version': { + if (!ownsLifecycleMutation()) break; + ds.cliVersion = msg.version; + dashboardEventBus.publish({ + type: 'session.update', + body: { + sessionId: ds.session.sessionId, + patch: { cliVersion: msg.version }, + }, + }); + break; + } + case 'runner_build_ready': { const identity = runtimeBuildIdentity(); if ( diff --git a/src/dashboard.ts b/src/dashboard.ts index 93f7cbff6..5a88d0549 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -1003,6 +1003,7 @@ interface ResolvedDashboardSettings { * source the SPA can offer as a one-click fill; never persisted unless picked. */ herdrTraexPlugin: { enabled: boolean; source: string; ref: string; recommendedSource: string; recommendedRef: string }; codexRpcInput: boolean; + autoUpgradeCodexSessions: boolean; /** Whether botmux auto-bypasses Codex's interactive hook-trust gate for * Codex-family plain-TUI launches. Default ON (only an explicit false disables). */ bypassCodexHookTrust: boolean; @@ -1604,6 +1605,7 @@ function resolveDashboardSettings(): ResolvedDashboardSettings { recommendedRef: TRAEX_RECOMMENDED_REF, }, codexRpcInput: dashboard.codexRpcInput === true, // default OFF until live-verified + autoUpgradeCodexSessions: dashboard.autoUpgradeCodexSessions !== false, // default ON — only an explicit stored false disables (matches config.ts getter) bypassCodexHookTrust: dashboard.bypassCodexHookTrust !== false, codexNotifier: { diff --git a/src/dashboard/settings-write-applier.ts b/src/dashboard/settings-write-applier.ts index 0a6a0bee2..189c1949e 100644 --- a/src/dashboard/settings-write-applier.ts +++ b/src/dashboard/settings-write-applier.ts @@ -50,6 +50,7 @@ export interface ResolvedDashboardSettingsView { chatBotDiscovery: boolean; herdrTraexPlugin: { enabled: boolean; source: string; ref: string; recommendedSource: string; recommendedRef: string }; codexRpcInput: boolean; + autoUpgradeCodexSessions: boolean; bypassCodexHookTrust: boolean; codexNotifier: { enabled: boolean; @@ -213,6 +214,7 @@ export type ApplySettingsWriteError = | 'invalid_herdrTraexPlugin_source' | 'invalid_herdrTraexPlugin_ref' | 'invalid_codexRpcInput' + | 'invalid_autoUpgradeCodexSessions' | 'invalid_bypassCodexHookTrust' | 'invalid_codexNotifier' | 'invalid_codexNotifier_enabled' @@ -431,6 +433,12 @@ export async function applySettingsWrite( } patch.codexRpcInput = obj.codexRpcInput; } + if ('autoUpgradeCodexSessions' in obj) { + if (typeof obj.autoUpgradeCodexSessions !== 'boolean') { + return { ok: false, error: 'invalid_autoUpgradeCodexSessions' }; + } + patch.autoUpgradeCodexSessions = obj.autoUpgradeCodexSessions; + } if ('bypassCodexHookTrust' in obj) { if (typeof obj.bypassCodexHookTrust !== 'boolean') { return { ok: false, error: 'invalid_bypassCodexHookTrust' }; diff --git a/src/dashboard/web/i18n.ts b/src/dashboard/web/i18n.ts index 3d178fe40..428e882c3 100644 --- a/src/dashboard/web/i18n.ts +++ b/src/dashboard/web/i18n.ts @@ -1489,6 +1489,8 @@ const zh = { 'settings.herdrTraexUnsupported': '当前 herdr 不支持插件；需 ≥ 0.7.0，请先运行 herdr update', 'settings.codexRpcInput': 'Codex 家族 RPC 输入模式', 'settings.codexRpcInputHelp': '实验性，默认关闭。对 codex / traex 机器人（仅 tmux 后端），用户消息经 app-server JSON-RPC 通道注入，绕开 codex 终端粘贴丢消息的问题（pane 仍跑真实 --remote TUI 渲染）。sandbox/隔离/审批门控/wrapper/带启动命令的会话自动回退传统粘贴。', + 'settings.autoUpgradeCodexSessions': '自动升级会话 Codex 版本', + 'settings.autoUpgradeCodexSessionsHelp': '默认开启。跟随本机已安装的新版 Codex，在安全空闲时更新会话进程并恢复原会话，不额外发起推理。后台任务或无法确认安全状态的会话会延后升级。开关实时生效。', 'settings.bypassCodexHookTrust': '自动信任 Codex Hook', 'settings.bypassCodexHookTrustHelp': '默认开启。Codex 0.14x 会弹交互式 hook 信任门（Press t to trust）；botmux 每次升级重写自装 hook 会让 hash 变化、门重新弹出，托管会话没人能按 t → 首条消息卡死。开启后对 codex / traex 普通 TUI 传 --dangerously-bypass-hook-trust。注意：该 flag 信任 codex 见到的所有 hook 来源（含项目 .codex/hooks.json 与已启用插件），不只 botmux 自装的；不想自动信任第三方项目/插件 hook 可关闭。受限机器人（关绕过）始终不受影响。下次会话生效，不影响已存活的 pane。', 'settings.codexNotifier': 'Codex 任务完成通知', @@ -4303,6 +4305,8 @@ const en: Record = { 'settings.herdrTraexUnsupported': 'This herdr does not support plugins; requires >= 0.7.0. Run herdr update first', 'settings.codexRpcInput': 'Codex-family RPC input mode', 'settings.codexRpcInputHelp': 'Experimental, off by default. For codex / traex bots (tmux backend only), user messages are injected via the app-server JSON-RPC channel, bypassing codex terminal paste-drops (the pane still runs the real --remote TUI). Sandbox/isolation/approval-gated/wrapper/startup-command sessions fall back to paste.', + 'settings.autoUpgradeCodexSessions': 'Automatically upgrade session Codex versions', + 'settings.autoUpgradeCodexSessionsHelp': 'On by default. Follow a newer Codex version already installed on this machine, replacing safely idle processes and resuming the same thread without additional inference. Defer sessions with background work or uncertain safety. Changes take effect live.', 'settings.bypassCodexHookTrust': 'Auto-trust Codex hooks', 'settings.bypassCodexHookTrustHelp': 'On by default. Codex 0.14x shows an interactive hook-trust gate ("Press t to trust"); every botmux upgrade rewrites its bundled hook so the hash changes and the gate re-fires, and a botmux-managed session has no one to press t → the first message wedges. When on, codex / traex plain-TUI launches pass --dangerously-bypass-hook-trust. Note: that flag trusts ALL hook sources codex sees (including a project .codex/hooks.json and enabled plugins), not only botmux\'s — turn it off if you do not want third-party project/plugin hooks auto-trusted. Restricted bots (bypass disabled) are never affected. Takes effect on the next session; running panes are unchanged.', 'settings.codexNotifier': 'Codex task completion notifications', diff --git a/src/dashboard/web/settings-page.tsx b/src/dashboard/web/settings-page.tsx index 64eb3e3f7..cb5b2699b 100644 --- a/src/dashboard/web/settings-page.tsx +++ b/src/dashboard/web/settings-page.tsx @@ -28,6 +28,7 @@ interface DashboardSettings { recommendedRef: string; }; codexRpcInput: boolean; + autoUpgradeCodexSessions: boolean; bypassCodexHookTrust: boolean; codexNotifier: { enabled: boolean; @@ -177,6 +178,7 @@ function parseSettings(s: any): DashboardSettings { recommendedRef: typeof s?.herdrTraexPlugin?.recommendedRef === 'string' ? s.herdrTraexPlugin.recommendedRef : '', }, codexRpcInput: s?.codexRpcInput === true, + autoUpgradeCodexSessions: s?.autoUpgradeCodexSessions !== false, // default ON — only an explicit persisted false disables (matches server snapshot) bypassCodexHookTrust: s?.bypassCodexHookTrust !== false, codexNotifier: { @@ -720,7 +722,7 @@ function SettingsBody(props: { const autoUpdateDisabled = !canWrite || settings.localDevInstall || !settings.autoUpdateSupported; const autoRestartDisabled = !canWrite || settings.maintenance.autoUpdate?.enabled !== true; - const saveBoolean = (key: 'publicReadOnly' | 'openTerminalInFeishu' | 'enableLocalCliOpen' | 'chatBotDiscovery' | 'codexRpcInput' | 'bypassCodexHookTrust' | 'noVisibleOutputHint' | 'remoteAccess', value: boolean) => { + const saveBoolean = (key: 'publicReadOnly' | 'openTerminalInFeishu' | 'enableLocalCliOpen' | 'chatBotDiscovery' | 'codexRpcInput' | 'autoUpgradeCodexSessions' | 'bypassCodexHookTrust' | 'noVisibleOutputHint' | 'remoteAccess', value: boolean) => { void props.onSave(key, { [key]: value }, s => ({ ...s, [key]: value })); }; const saveHerdrTraexPlugin = (patch: Partial>) => { @@ -863,6 +865,13 @@ function SettingsBody(props: { disabled={dis || savingKey === 'codexRpcInput'} onChange={value => saveBoolean('codexRpcInput', value)} /> + saveBoolean('autoUpgradeCodexSessions', value)} + /> (); + +export interface CodexExecutable { + path: string; + version: string; + fingerprint: string; +} + +export interface CodexProcess extends CodexExecutable { + pid: number; + started: string; +} + +/** This probe only runs --version; it must never open a thread or generate text. */ +export async function probeCodexExecutable(path: string): Promise { + const real = realpathSync(path); + const stat = statSync(real); + const { stdout } = await exec(real, ['--version'], { timeout: 5_000, maxBuffer: 16_384 }); + const version = /^codex-cli\s+(\d+\.\d+\.\d+(?:[-+][\w.-]+)?)/m.exec(stdout)?.[1]; + if (!version) throw new Error('Executable did not identify itself as Codex'); + return { path: real, version, fingerprint: `${real}:${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeMs}` }; +} + +export interface ProcessRow { pid: number; ppid: number; command: string } + +export function parseUpgradeProcessTable(text: string): ProcessRow[] { + return text.split('\n').flatMap(line => { + const match = /^\s*(\d+)\s+(\d+)\s+(.+)$/.exec(line); + return match ? [{ pid: Number(match[1]), ppid: Number(match[2]), command: match[3]!.trim() }] : []; + }); +} + +export function upgradeProcessTree(rows: readonly ProcessRow[], roots: readonly number[]): ProcessRow[] { + const included = new Set(roots); + let changed = true; + while (changed) { + changed = false; + for (const row of rows) { + if (included.has(row.ppid) && !included.has(row.pid)) { + included.add(row.pid); + changed = true; + } + } + } + return rows.filter(row => included.has(row.pid)); +} + +/** Failure is unknown, never an empty (apparently idle) process tree. */ +export async function readUpgradeProcessTable(): Promise { + const { stdout } = await exec('/bin/ps', ['-axo', 'pid=,ppid=,comm='], { + timeout: 5_000, maxBuffer: 4 * 1024 * 1024, + }); + const rows = parseUpgradeProcessTable(stdout); + if (!rows.length) throw new Error('Unable to inspect Codex process tree'); + return rows; +} + +export async function probeRunningCodex(row: ProcessRow): Promise { + const started = readProcessStartIdentity(row.pid); + if (!started) throw new Error('Codex process identity is unavailable'); + const observed = observedProcesses.get(row.pid); + if (observed?.started === started) return observed; + // Linux /proc exposes the *running inode*, even after an installer unlinks it. + // Executing a newly resolved PATH binary would report the wrong generation. + const path = process.platform === 'linux' ? `/proc/${row.pid}/exe` : row.command; + let executable: CodexExecutable; + if (process.platform === 'linux') { + const { stdout } = await exec(path, ['--version'], { timeout: 5_000, maxBuffer: 16_384 }); + const version = /^codex-cli\s+(\d+\.\d+\.\d+(?:[-+][\w.-]+)?)/m.exec(stdout)?.[1]; + if (!version) throw new Error('Running process did not identify itself as Codex'); + const stat = statSync(path); + executable = { path: readlinkSync(path), version, fingerprint: `${stat.dev}:${stat.ino}` }; + } else { + executable = await probeCodexExecutable(path); + // An in-place replacement cannot tell us the old process's version on macOS. + // Leave it alone instead of calling the replacement's version "running". + const born = Date.parse(started); + if (!Number.isFinite(born) || statSync(executable.path).mtimeMs > born + 1_000) { + throw new Error('Running Codex executable changed after process start'); + } + } + if (readProcessStartIdentity(row.pid) !== started) throw new Error('Codex process changed during inspection'); + const observedProcess = { ...executable, pid: row.pid, started }; + observedProcesses.set(row.pid, observedProcess); + return observedProcess; +} + +export function isCodexProcess(row: ProcessRow): boolean { + return basename(row.command) === 'codex'; +} + +/** A live npm gateway may predate the current global launcher. Match its + * installed platform package and the owning Botmux package, not just its name. */ +function isInstalledBotmuxBinary(executable: string): boolean { + if (basename(executable) !== 'botmux') return false; + const platformDir = dirname(executable); + const packageName = basename(platformDir); + if (!/^botmux-(?:darwin-(?:arm64|x64)|linux-(?:arm64|x64)(?:-musl)?)$/.test(packageName) + || basename(dirname(platformDir)) !== 'node_modules') return false; + const binaryPackage = JSON.parse(readFileSync(join(platformDir, 'package.json'), 'utf8')); + const ownerPackage = JSON.parse(readFileSync(join(dirname(dirname(platformDir)), 'package.json'), 'utf8')); + return binaryPackage.name === packageName && typeof binaryPackage.version === 'string' + && ownerPackage.name === 'botmux' && ownerPackage.version === binaryPackage.version + && ownerPackage.optionalDependencies?.[packageName] === binaryPackage.version; +} + +/** Only known, stateless leaf helpers may be replaced with the idle CLI. + * A generic node process, user MCP, REPL or helper with children is unknown. */ +export function isRestartableCodexHelper( + row: ProcessRow, tree: readonly ProcessRow[], running: readonly CodexProcess[], gatewayCommand: string, +): boolean { + if (!running.some(parent => parent.pid === row.ppid) + || tree.some(child => child.ppid === row.pid)) return false; + const started = readProcessStartIdentity(row.pid); + if (!started) return false; + const argv = readCmdline(row.pid); + let known = false; + try { + const executable = realpathSync(process.platform === 'linux' ? `/proc/${row.pid}/exe` : row.command); + if (argv.length === 1 && running.some(parent => parent.pid === row.ppid + && executable === join(dirname(parent.path), 'codex-code-mode-host'))) { + known = true; + } else if (argv.length === 3 && argv[1] === 'mcp' && argv[2] === 'serve') { + const gateway = realpathSync(gatewayCommand); + known = executable === gateway; + // Standalone installs use this exact two-line launcher. Inspect only a + // small wrapper, never read a compiled binary's entire payload or run sh. + if (!known && statSync(gateway).size <= 16_384) { + const target = /^#!\/bin\/sh\nexec "(\/[^"\\$`\r\n]+)" "\$@"\n?$/.exec(readFileSync(gateway, 'utf8'))?.[1]; + known = !!target && executable === realpathSync(target); + } + if (!known) known = isInstalledBotmuxBinary(executable); + } else if (argv.length === 4 && basename(executable) === 'node' + && argv[2] === 'mcp' && argv[3] === 'serve') { + const script = parseWrapperCliEntry(readFileSync(gatewayCommand, 'utf8')); + known = !!script && realpathSync(argv[1]!) === realpathSync(script); + } + } catch { + // No identity evidence means the helper is not eligible for replacement. + return false; + } + return known && readProcessStartIdentity(row.pid) === started; +} + +export function upgradeRequired(installed: CodexExecutable, running: readonly CodexProcess[]): boolean { + return running.length > 0 + && !running.some(current => isNewerVersion(current.version, installed.version)) + && running.some(current => isNewerVersion(installed.version, current.version)); +} + +/** Native goals can resume themselves without Botmux submitting a prompt. + * Until their quiescence can be proven over every supported TUI protocol, + * leave these sessions running. Do not infer completion from screen silence. */ +export async function hasCodexAutonomousGoal(path: string): Promise { + const stream = createReadStream(path); + const lines = createInterface({ input: stream, crlfDelay: Infinity }); + try { + for await (const line of lines) { + if (!line.trim()) continue; + const entry = JSON.parse(line) as { type?: string; payload?: Record }; + const payload = entry.payload; + if (!payload) continue; + if (entry.type === 'event_msg' && typeof payload.type === 'string' && /goal/i.test(payload.type)) return true; + if (entry.type === 'response_item' && payload.type === 'function_call' + && typeof payload.name === 'string' && /(?:create_goal|goal[./]set)$/.test(payload.name)) return true; + } + return false; + } finally { + lines.close(); + stream.destroy(); + } +} + +export async function waitForCodexExit(processes: readonly CodexProcess[], timeoutMs = 8_000): Promise { + const deadline = Date.now() + timeoutMs; + while (true) { + const alive = processes.filter(item => { + try { process.kill(item.pid, 0); } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') return false; + throw error; + } + const started = readProcessStartIdentity(item.pid); + if (!started) throw new Error('Unable to confirm retired Codex process identity'); + return started === item.started; + }); + if (!alive.length) return; + if (Date.now() >= deadline) throw new Error('Old Codex process is still alive; replacement was not started'); + await new Promise(resolve => setTimeout(resolve, 100)); + } +} + +export type CodexUpgradeState = 'current' | 'waiting' | 'upgrading' | 'failed'; + +/** Serializes observations. The worker owns the input queues throughout a swap. */ +export class CodexSessionUpgradeMonitor { + private inFlight = false; + private lastStatus = ''; + constructor(private readonly deps: { + enabled: () => boolean; + check: () => Promise<{ target: CodexExecutable; running: CodexProcess[] } | undefined>; + blocked: () => string | undefined; + upgrade: (target: CodexExecutable, running: CodexProcess[]) => Promise; + report: (state: CodexUpgradeState, reason: string) => void; + }) {} + + private report(state: CodexUpgradeState, reason: string): void { + const key = `${state}:${reason}`; + if (this.lastStatus === key) return; + this.lastStatus = key; + this.deps.report(state, reason); + } + + async tick(): Promise { + if (this.inFlight || !this.deps.enabled()) return; + this.inFlight = true; + try { + const snapshot = await this.deps.check(); + if (!snapshot || !this.deps.enabled()) return; + if (!upgradeRequired(snapshot.target, snapshot.running)) return; + const reason = this.deps.blocked(); + if (reason) { this.report('waiting', reason); return; } + this.report('upgrading', snapshot.target.version); + await this.deps.upgrade(snapshot.target, snapshot.running); + this.report('current', snapshot.target.version); + } catch (error) { + this.report('failed', error instanceof Error ? error.message : String(error)); + } finally { + this.inFlight = false; + } + } +} diff --git a/src/services/codex-upgrade-target.ts b/src/services/codex-upgrade-target.ts new file mode 100644 index 000000000..7347ea267 --- /dev/null +++ b/src/services/codex-upgrade-target.ts @@ -0,0 +1,46 @@ +import { lstatSync, realpathSync, statSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join, relative, sep } from 'node:path'; +import { resolveCommandReal } from '../adapters/cli/registry.js'; + +export interface CodexUpgradeCommandOptions { + cliPathOverride?: string; + cliRuntime?: { source: string }; + wrapperCli?: string; +} + +/** Select the installed official release without letting an older PATH entry + * mask the managed current pointer. Explicit runtime selections keep their + * existing resolver; this never downloads or changes an installation. */ +export function resolveCodexUpgradeCommand(options: CodexUpgradeCommandOptions = {}): string { + if (options.cliPathOverride !== undefined + || (options.cliRuntime && options.cliRuntime.source !== 'official')) { + return resolveCommandReal(options.cliPathOverride ?? 'codex'); + } + + // Installation belongs to the host user, not a bot's redirected CODEX_HOME. + const home = homedir(); + const standalone = join(home, '.codex', 'packages', 'standalone'); + const current = join(standalone, 'current'); + try { + lstatSync(current); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return resolveCommandReal('codex'); + throw new Error('Cannot inspect the managed Codex current installation', { cause: error }); + } + + try { + const target = realpathSync(join(current, 'bin', 'codex')); + const entry = realpathSync(join(home, '.local', 'bin', 'codex')); + if (entry !== target) throw new Error('The user Codex entry does not point to managed current'); + const releasePath = relative(realpathSync(join(standalone, 'releases')), target).split(sep); + if (releasePath.length !== 3 || !releasePath[0] || releasePath[0] === '..' + || releasePath[1] !== 'bin' || releasePath[2] !== 'codex' + || !statSync(target).isFile()) { + throw new Error('Managed current is not a releases//bin/codex executable'); + } + return resolveCommandReal(target); + } catch (error) { + throw new Error(`Cannot select the managed Codex upgrade target: ${error instanceof Error ? error.message : String(error)}`, { cause: error }); + } +} diff --git a/src/types.ts b/src/types.ts index 8bae84206..abf503ed6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1440,6 +1440,7 @@ export type WorkerToDaemon = * read bridge send markers or emit transcript fallback for this session. */ | { type: 'session_close_ready'; sessionId: string } | { type: 'prompt_ready' } + | { type: 'cli_runtime_version'; version: string } | { type: 'runner_build_ready'; runnerBuildId: string } | { type: 'restart_result'; diff --git a/src/worker.ts b/src/worker.ts index 59955b74f..b7182c71e 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -270,6 +270,13 @@ import { resolveRenderDimensions, } from './utils/render-dimensions.js'; import { createCliAdapterSync, locateOnPath } from './adapters/cli/registry.js'; +import { resolveCodexUpgradeCommand } from './services/codex-upgrade-target.js'; +import { + CodexSessionUpgradeMonitor, hasCodexAutonomousGoal, isCodexProcess, isRestartableCodexHelper, + probeCodexExecutable, probeRunningCodex, readUpgradeProcessTable, upgradeProcessTree, + upgradeRequired, waitForCodexExit, + type CodexExecutable, type CodexProcess, +} from './services/codex-session-upgrade.js'; import { buildWrappedLaunch, parseWrapperCli, isTtadkWrapper, wrapperLaunchEnv } from './setup/cli-selection.js'; import { cliUnavailableMessage } from './setup/cli-availability.js'; import { @@ -2216,6 +2223,182 @@ let cliRestartInProgress = false; * type-ahead fallback. Keep them fenced across an owned restart until the * replacement generation reaches its prompt. */ let rawInputRestartGate = false; +let codexAutoUpgrade: { + stage: 'stopping' | 'restoring' | 'failed'; + threadId: string; + ready?: () => void; + reject?: (error: Error) => void; +} | undefined; +let codexUpgradeTimer: ReturnType | undefined; +let codexUpgradeInspectionBlock: string | undefined; +let codexRuntimeObservedGeneration = -1; + +function codexUpgradeBlocked(): string | undefined { + if (!lastInitConfig || !backend || !isPromptReady || awaitingFirstPrompt) return 'waiting for a ready session'; + if (lastInitConfig.adoptMode || lastInitConfig.existingAppServerEndpoint) return 'externally owned session'; + if (lastInitConfig.wrapperCli?.trim()) return 'a configured CLI wrapper controls the runtime'; + if (cliRestartInProgress || codexAutoUpgrade || tmuxRestartTimer) return 'another restart is in progress'; + if (isFlushing || injectionFlushing || commandLineWritesPending || initialInputOwnershipPending + || pendingMessages.length || pendingRawInputs.length || pendingInjections.length + || pendingAdoptMessages.length || sessionRenameInFlight() || pendingSessionRename + || durableTurnInFlight || tuiPromptBlocking || hookReviewInputHold || bareShellCheckInProgress + || ambiguousSubmissionRecoveryHold || submitFailureChains.size() + || codexAppTurnLiveness.hasActiveTurn() || codexAppCompletionAwaitingFinal + || codexAppTurnDispatchQueue.size() || codexAppRecoveredDispatches.length + || hasStructuredLifecycleBlock()) return 'waiting for the current turn and input queues'; + // Web terminal writes can bypass normal input queues. Never replace a CLI + // while a terminal client is attached, including during a pending handshake. + if (wsClients.size || clientPtys.size) return 'a Web Terminal is attached'; + return codexUpgradeInspectionBlock; +} + +async function inspectCodexUpgradeRuntime(): Promise { + const roots = [currentCodexObservedPid(), codexRpcEngine?.appServerPid] + .filter((pid): pid is number => typeof pid === 'number' && pid > 1); + if (!roots.length) throw new Error('Codex runtime PID is not available'); + const tree = upgradeProcessTree(await readUpgradeProcessTable(), roots); + const native = tree.filter(isCodexProcess); + if (!native.length) throw new Error('Codex runtime process is not observable'); + return Promise.all(native.map(probeRunningCodex)); +} + +async function observeCodexRuntimeVersionOnReady(): Promise { + const versionObservationBackend = backend; + const observedGeneration = cliSpawnGeneration; + const running = await inspectCodexUpgradeRuntime(); + // A late observation must not label another backend generation, or bypass + // the upgrade's own runtime/thread verification before it releases input. + if (!versionObservationBackend || backend !== versionObservationBackend || cliSpawnGeneration !== observedGeneration + || cliRestartInProgress || codexAutoUpgrade) return; + const version = running[0]?.version; + if (!version || running.some(item => item.version !== version)) { + throw new Error('Running Codex processes do not identify a single version'); + } + send({ type: 'cli_runtime_version', version }); +} + +async function autoUpgradeCodex(target: CodexExecutable, running: CodexProcess[]): Promise { + // No await may separate this final safety check and closing the input gates. + const blocked = codexUpgradeBlocked(); + if (blocked || !config.autoUpgradeCodexSessions) throw new Error(blocked ?? 'automatic upgrade disabled'); + const cfg = lastInitConfig!; + const threadId = remoteThreadId ?? cfg.cliSessionId; + if (!threadId) throw new Error('An exact Codex thread ID is required for automatic upgrade'); + codexAutoUpgrade = { stage: 'stopping', threadId }; + cliRestartInProgress = true; + rawInputRestartGate = true; + cliSpawnGeneration++; + const oldBackend = backend!; + intentionalRestartBackend = oldBackend; + let readyTimer: ReturnType | undefined; + try { + await Promise.race([ + Promise.resolve(oldBackend.destroySession?.()), + new Promise((_, reject) => { + readyTimer = setTimeout(() => reject(new Error('Old Codex teardown timed out')), 22_000); + }), + ]); + if (readyTimer) clearTimeout(readyTimer); + killCli({ preservePending: true, preservePolicyCapability: true, preserveInjections: true }); + await waitForCodexExit(running); + codexAutoUpgrade.stage = 'restoring'; + const ready = new Promise((resolve, reject) => { + codexAutoUpgrade!.ready = resolve; + codexAutoUpgrade!.reject = reject; + readyTimer = setTimeout(() => reject(new Error('New Codex did not become ready')), 90_000); + }); + // Attach rejection handling before spawn, which can itself fail or emit an + // exit notification. The original error is still awaited below. + void ready.catch(() => undefined); + const restartCfg = { + ...cfg, resume: true, prompt: '', cliSessionId: threadId, forkSession: false, + cliPathOverride: target.path, model: undefined, reasoningEffort: undefined, + nativeSessionTitle: undefined, nativeSessionTitlePrompt: undefined, + }; + let pluginGenerationPrepared = false; + if (codexRpcEligible(restartCfg, { sandboxForced: sandboxEnabled() })) { + await prepareCliPluginGenerationAndGateway(restartCfg, createCliAdapterSync(restartCfg.cliId as CliId, target.path)); + pluginGenerationPrepared = true; + const outcome = await engageCodexRpc(restartCfg); + if (outcome !== 'resumed' || remoteThreadId !== threadId) { + throw new Error('Codex RPC did not resume the original thread'); + } + } + awaitingFirstPrompt = true; + startScreenUpdates(); + startStuckDetector(); + replacementSpawnInProgress = true; + await spawnCli(restartCfg, { pluginGenerationPrepared }); + await ready; + const replacement = await inspectCodexUpgradeRuntime(); + const resumedOriginal = replacement.some(item => { + const open = findCodexRolloutSetByPid(item.pid); + return open?.size === 1 && open.has(threadId.toLowerCase()); + }); + if (replacement.some(item => item.version !== target.version) + || !resumedOriginal || (remoteThreadId ?? lastInitConfig?.cliSessionId) !== threadId) { + throw new Error('Replacement Codex runtime or thread identity did not match'); + } + if (cfg.workingDir !== restartCfg.workingDir || cfg.env !== restartCfg.env) { + throw new Error('Session launch settings changed during upgrade; restart is required'); + } + send({ type: 'cli_runtime_version', version: target.version }); + codexAutoUpgrade = undefined; + cliRestartInProgress = false; + releaseRawInputRestartGate(); + void flushPendingInjections(); + void flushPending(); + } catch (error) { + // Keep the worker and its accepted queues alive. Automatic crash recovery + // can demote resume to a fresh thread, so it is not a valid upgrade fallback. + codexAutoUpgrade = { stage: 'failed', threadId }; + cliRestartInProgress = true; + rawInputRestartGate = true; + send({ type: 'user_notify', message: `Codex 会话自动升级未完成，原会话和待处理消息已保留。请检查后在当前会话发送 /restart 重启。原因：${error instanceof Error ? error.message : String(error)}` }); + throw error; + } finally { + replacementSpawnInProgress = false; + if (readyTimer) clearTimeout(readyTimer); + } +} + +const codexUpgradeMonitor = new CodexSessionUpgradeMonitor({ + enabled: () => config.autoUpgradeCodexSessions + && (lastInitConfig?.cliId === 'codex' || lastInitConfig?.cliId === 'codex-app') + && !lastInitConfig?.adoptMode && !lastInitConfig?.existingAppServerEndpoint + && !lastInitConfig?.wrapperCli?.trim() + && !codexAutoUpgrade && !cliRestartInProgress, + check: async () => { + if (!backend || !isPromptReady || awaitingFirstPrompt) return undefined; + const checkedGeneration = cliSpawnGeneration; + const target = await probeCodexExecutable(resolveCodexUpgradeCommand(lastInitConfig!)); + const running = await inspectCodexUpgradeRuntime(); + if (!upgradeRequired(target, running)) return undefined; + codexUpgradeInspectionBlock = undefined; + const roots = running.map(item => item.pid); + const tree = upgradeProcessTree(await readUpgradeProcessTable(), roots); + if (tree.some(row => !isCodexProcess(row) + && !isRestartableCodexHelper(row, tree, running, defaultGatewayEntry().command))) { + codexUpgradeInspectionBlock = 'background processes are still attached to Codex'; + } + const threadId = remoteThreadId ?? lastInitConfig?.cliSessionId; + const owner = codexRpcEngine?.appServerPid + ?? running.find(item => findCodexRolloutSetByPid(item.pid)?.has(threadId?.toLowerCase() ?? ''))?.pid; + const open = owner ? findCodexRolloutSetByPid(owner) : undefined; + if (!threadId || !open || open.size !== 1 || !open.has(threadId.toLowerCase())) { + codexUpgradeInspectionBlock = 'waiting for exclusive ownership of the original thread'; + } + const rollout = owner ? findCodexRolloutByPid(owner) : undefined; + if (!rollout || await hasCodexAutonomousGoal(rollout.path)) { + codexUpgradeInspectionBlock = 'native Goal state cannot be safely resumed without inference'; + } + if (checkedGeneration !== cliSpawnGeneration) return undefined; + return { target, running }; + }, + blocked: codexUpgradeBlocked, + upgrade: autoUpgradeCodex, + report: (state, reason) => log(`Codex session upgrade ${state}: ${reason}`), +}); /** Per-spawn one-shot: have this spawn's bot.startupCommands been typed in yet? * Reset in spawnCli so a restart/resume (which re-spawns the CLI) re-applies * them — needed because session-only settings like `/effort ultracode` are lost @@ -3343,6 +3526,12 @@ function failCodexAppControlGeneration(reason: string): void { } try { backend?.kill(); } catch { /* process exit is the final fail-close */ } backend = null; + if (codexAutoUpgrade) { + // The rejected replacement stays unusable, but the worker must retain + // messages accepted during an automatic version change. + codexAutoUpgrade.reject?.(new Error(reason)); + return; + } queueMicrotask(() => { void sendFatalWorkerErrorAndExit( new Error(reason), @@ -10729,6 +10918,13 @@ function markPromptReady(): void { renderer?.markNewTurn(); // exclude history replay from streaming card } send({ type: 'prompt_ready' }); + if (codexAutoUpgrade?.stage === 'restoring') codexAutoUpgrade.ready?.(); + if (codexUpgradeTimer && codexRuntimeObservedGeneration !== cliSpawnGeneration) { + // Record the actual running version while its executable is still present. + // macOS cannot recover an unlinked old binary after an installer replaces it. + codexRuntimeObservedGeneration = cliSpawnGeneration; + void observeCodexRuntimeVersionOnReady().catch(error => log(`Codex runtime observation unavailable: ${error.message}`)); + } if ( persistCodexRunnerBuildOnReady && lastInitConfig?.cliId === 'codex-app' @@ -13972,6 +14168,9 @@ async function spawnCli( if (spawnGeneration !== cliSpawnGeneration) throw new CliSpawnSupersededError(); const replacementExpectedFresh = codexRunnerFreshness === 'restarting_fresh'; + if (codexAutoUpgrade?.stage === 'restoring' && willReattachPersistent) { + throw new Error('Automatic Codex upgrade refused to reattach an old persistent process'); + } const freshness = decideCodexRunnerFreshness({ cliId: cfg.cliId, adoptMode: cfg.adoptMode === true, @@ -14010,7 +14209,8 @@ async function spawnCli( // Fresh spawns (incl. resume that starts a new CLI, where hasSession is false) // arm it. spawnCli is synchronous up to backend spawn, so this lands before // any flushPending consumes the flag. - hasRunStartupCommands = !shouldRunStartupCommandsOnSpawn({ willReattachPersistent }); + hasRunStartupCommands = !shouldRunStartupCommandsOnSpawn({ willReattachPersistent }) + || codexAutoUpgrade?.stage === 'restoring'; // Re-arm the bare-shell launch detector for this spawn (fresh OR reattach). It // runs once on the first flush and only fires when the pane leaf is actually a // bare shell, so a healthy reattach (leaf = the live CLI) self-excludes while a @@ -14096,7 +14296,8 @@ async function spawnCli( { botmuxSessionProfile: basename(cfg.cliPathOverride ?? '') === 'hermes-botmux-session' }, ) : undefined; - const tier2ForceFresh = effectiveResume && consecutiveInWorkerRestarts >= 2; + const tier2ForceFresh = effectiveResume && consecutiveInWorkerRestarts >= 2 + && codexAutoUpgrade?.stage !== 'restoring'; // Tier 0: see block comment above. The adapter itself drops --resume when // the id is missing (its buildArgs starts fresh); demoting HERE keeps // effectiveResume honest so the fresh-demotion notice below fires. @@ -14117,6 +14318,10 @@ async function spawnCli( } const fallBackToFresh = effectiveResume && !willReattachPersistent && (tier1ProbeFalse || tier2ForceFresh || missingExactResumeId); + if (codexAutoUpgrade?.stage === 'restoring' + && (fallBackToFresh || !effectiveResume || effectiveCliSessionId !== codexAutoUpgrade.threadId)) { + throw new Error('Automatic Codex upgrade requires the original resume target; fresh fallback is disabled'); + } if (fallBackToFresh) { const reason = tier2ForceFresh ? `consecutive restart x${consecutiveInWorkerRestarts} — 2nd failed resume attempt` @@ -14320,6 +14525,7 @@ async function spawnCli( resume: effectiveResume, workingDir: buildArgsWorkingDir, resumeSessionId: effectiveCliSessionId, + quietResume: codexAutoUpgrade?.stage === 'restoring', // Native session fork (Claude --fork-session / codex fork): resume the // source transcript but branch into a fresh CLI-minted id. Only on the // child's first spawn (cfg.forkSession) AND only when we actually resume — @@ -16478,6 +16684,16 @@ async function spawnCli( log(`Ignored stale backend exit (code: ${code}, signal: ${signal})`); return; } + if (codexAutoUpgrade) { + backend = null; + isPromptReady = false; + if (codexAutoUpgrade.stage === 'restoring') { + codexAutoUpgrade.reject?.(new Error(`Replacement Codex exited before readiness (${code ?? signal})`)); + } + // The upgrade owns recovery; a daemon crash-restart could lose the + // original thread or replay messages that this worker still owns. + return; + } const recoveryHeld = ambiguousSubmissionRecoveryHold; const handedOffDurable = handoffQueuedDurableInputsOnBackendExit( pendingMessages, @@ -16790,6 +17006,7 @@ function restoreMojoLivePatchAfterRespawn(): void { function killCli(opts: { preservePending?: boolean; + preserveInjections?: boolean; /** An intentional in-worker CLI restart replaces only the live backend. The * surviving Node worker keeps its generation-scoped policy authority. */ preservePolicyCapability?: boolean; @@ -16891,11 +17108,11 @@ function killCli(opts: { // because /rename owned the TUI or an owned CLI restart was already fenced. // Preserve them across restart; unlike an in-flight raw command, replaying // these cannot duplicate a side effect. - // pendingInjections 则无条件清空（不随 preservePending 保留）：barrier /cd 的 + // 普通重启清空 pendingInjections（不随 preservePending 保留）：barrier /cd 的 // 目录变更已固化进 lastInitConfig.workingDir 与 daemon 记录，respawn 本身就落在 // 新目录，重放 /cd 反而多余；非 barrier 注入（如 /compact）是 best-effort， - // 不跨进程重放。 - pendingInjections.length = 0; + // 不跨进程重放。自动升级保留换代窗口内尚未写入的注入。 + if (!opts.preserveInjections) pendingInjections.length = 0; scrollback = ''; herdrWebHistory = null; herdrWebScrollDirection = null; @@ -16924,6 +17141,10 @@ async function restartCliProcess( reason: string, opts: { immediate?: boolean; preservePending?: boolean; skipRestartBudget?: boolean } = {}, ): Promise { + if (codexAutoUpgrade) { + codexAutoUpgrade.reject?.(new Error(reason)); + return; + } if (lastInitConfig?.adoptMode || lastInitConfig?.existingAppServerEndpoint) { log(`Restart ignored in shared-adopt mode (${reason})`); return; @@ -17180,6 +17401,10 @@ function startWebServer(host: string, preferredPort?: number): Promise { // `connection` callback is too late: an unauthenticated localhost scanner // briefly becomes a client and races the terminal history seed. verifyClient: ({ req }, done) => { + if (codexAutoUpgrade) { + done(false, 503, 'Codex session is restarting'); + return; + } const url = parseWorkerRequestUrl(req); if (!url) { done(false, 400, 'Bad Request'); @@ -17194,6 +17419,10 @@ function startWebServer(host: string, preferredPort?: number): Promise { }); wss.on('connection', (ws, req: IncomingMessage) => { + if (codexAutoUpgrade) { + ws.close(1013, 'Codex session is restarting'); + return; + } // P1-3 — the pre-handshake verdict is a SNAPSHOT, and this callback runs // strictly after it: ws only completes the upgrade once verifyClient // called back, and re-resolving access here re-reads `.dashboard-secret` @@ -19201,6 +19430,11 @@ process.on('message', async (raw: unknown) => { } } lastInitConfig = msg; + if (!codexUpgradeTimer && (msg.cliId === 'codex' || msg.cliId === 'codex-app') + && !msg.adoptMode && !msg.existingAppServerEndpoint) { + codexUpgradeTimer = setInterval(() => { void codexUpgradeMonitor.tick(); }, 60_000); + codexUpgradeTimer.unref(); + } initialInputOwnershipPending = !!msg.prompt; activeRestartAttemptId = msg.restartAttemptId; sessionId = msg.sessionId; @@ -19729,6 +19963,10 @@ process.on('message', async (raw: unknown) => { } case 'restart': { + if (codexAutoUpgrade?.stage === 'failed') { + codexAutoUpgrade = undefined; + cliRestartInProgress = false; + } if (effectiveBackendType === 'riff') { log('Refused Riff generation restart; the existing lineage-owning worker is retained'); break; @@ -20576,6 +20814,8 @@ process.on('message', async (raw: unknown) => { // ─── Cleanup ───────────────────────────────────────────────────────────────── function cleanup(): void { + if (codexUpgradeTimer) clearInterval(codexUpgradeTimer); + codexUpgradeTimer = undefined; stopNativeSessionTitleSync(); cleanupPiInitialPromptFiles(); stopSessionMcpGatewayHost(); diff --git a/test/auto-upgrade-codex-sessions-config.test.ts b/test/auto-upgrade-codex-sessions-config.test.ts new file mode 100644 index 000000000..6a4d2f41f --- /dev/null +++ b/test/auto-upgrade-codex-sessions-config.test.ts @@ -0,0 +1,43 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { config } from '../src/config.js'; +import { globalConfigPath, invalidateGlobalConfigCache, mergeDashboardConfig, readGlobalConfig } from '../src/global-config.js'; + +describe('automatic Codex session upgrades (default ON)', () => { + let configDir: string; + + beforeEach(() => { + configDir = mkdtempSync(join(tmpdir(), 'botmux-codex-session-upgrade-')); + vi.stubEnv('HOME', configDir); + mkdirSync(dirname(globalConfigPath()), { recursive: true }); + invalidateGlobalConfigCache(); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + invalidateGlobalConfigCache(); + rmSync(configDir, { recursive: true, force: true }); + }); + + it('enables upgrades when no configuration has been saved', () => { + expect(config.autoUpgradeCodexSessions).toBe(true); + }); + + it('reads saved disable and re-enable changes without reloading the runtime configuration', () => { + mergeDashboardConfig({ autoUpgradeCodexSessions: false, codexRpcInput: true }); + expect(config.autoUpgradeCodexSessions).toBe(false); + expect(readGlobalConfig().dashboard?.autoUpgradeCodexSessions).toBe(false); + + mergeDashboardConfig({ autoUpgradeCodexSessions: true }); + expect(config.autoUpgradeCodexSessions).toBe(true); + expect(readGlobalConfig().dashboard?.codexRpcInput).toBe(true); + }); + + it.each([{}, { autoUpgradeCodexSessions: 'false' }])('defaults ON for an absent or invalid saved value: %j', (dashboard) => { + writeFileSync(globalConfigPath(), JSON.stringify({ dashboard })); + expect(config.autoUpgradeCodexSessions).toBe(true); + expect(readGlobalConfig().dashboard?.autoUpgradeCodexSessions).toBeUndefined(); + }); +}); diff --git a/test/cli-adapters.test.ts b/test/cli-adapters.test.ts index 790b56088..a1c34171a 100644 --- a/test/cli-adapters.test.ts +++ b/test/cli-adapters.test.ts @@ -606,6 +606,22 @@ describe('codex buildArgs', () => { expect(configIdx).toBeLessThan(args.indexOf('codex-session-id')); }); + it('disables automatic recap for a quiet resume without adding a prompt', () => { + const normal = adapter.buildArgs({ sessionId: 'sess-quiet', resume: true, resumeSessionId: 'codex-existing' }); + const quiet = adapter.buildArgs({ sessionId: 'sess-quiet', resume: true, resumeSessionId: 'codex-existing', quietResume: true }); + expect(normal).not.toContain('tui.auto_recap=false'); + expect(quiet).toEqual([...normal.slice(0, -1), '-c', 'tui.auto_recap=false', 'codex-existing']); + expect(adapter.buildArgs({ sessionId: 'sess-quiet', resume: false, quietResume: true })).not.toContain('tui.auto_recap=false'); + }); + + it('also suppresses automatic recap when quietly attaching the RPC viewer', () => { + const args = adapter.buildArgs({ + sessionId: 'sess-quiet-rpc', resume: true, quietResume: true, + remoteWsUrl: 'ws://127.0.0.1:9933', remoteThreadId: 'thread-existing', + }); + expect(args.slice(-3)).toEqual(['-c', 'tui.auto_recap=false', 'thread-existing']); + }); + it('passes configured model with --model', () => { const args = adapter.buildArgs({ sessionId: 'sess-4', resume: false, model: 'gpt-5-codex' }); const idx = args.indexOf('--model'); @@ -646,6 +662,13 @@ describe('codex-app buildArgs', () => { }); expect(args).toContain('--thread-id'); expect(args).toContain('thread-123'); + expect(args).not.toContain('--strict-resume'); + }); + + it('requires strict thread resume for a quiet maintenance restore without injecting a prompt', () => { + const normal = adapter.buildArgs({ sessionId: 'sess-app', resume: true, resumeSessionId: 'thread-123' }); + const quiet = adapter.buildArgs({ sessionId: 'sess-app', resume: true, resumeSessionId: 'thread-123', quietResume: true }); + expect(quiet).toEqual([...normal, '--strict-resume']); }); it('canonicalizes a symlinked codex so --codex-bin matches the sandbox-authorized path', () => { diff --git a/test/codex-app-runner.integration.test.ts b/test/codex-app-runner.integration.test.ts index 0f2252c9e..545c95610 100644 --- a/test/codex-app-runner.integration.test.ts +++ b/test/codex-app-runner.integration.test.ts @@ -2163,6 +2163,106 @@ describe('codex-app-runner app-server protocol integration', { timeout: 120_000, } }); + it('strictly resumes the original thread to ready with no input, inference request, or usage output', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-quiet-resume-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir); + await control.listen(); + const harness = startRunner( + fakeCodex, dir, logPath, '0.153.4', 'success', control.bootstrap.path, + { + threadId: 'thread-existing', + extraArgs: ['--strict-resume'], + // Any accidental turn would produce a usage-bearing final in this fixture. + env: { FAKE_TOKEN_USAGE: '1' }, + }, + ); + try { + await waitFor(harness, () => control.states.some(state => state.busy === false)); + // Observe the ready runner without sending even an empty input record. + await new Promise(resolvePromise => setTimeout(resolvePromise, 150)); + const requests = readRequests(logPath); + expect(requests.filter(request => request.method === 'thread/resume')).toEqual([ + expect.objectContaining({ params: expect.objectContaining({ threadId: 'thread-existing' }) }), + ]); + expect(requests.filter(request => ['thread/start', 'turn/start', 'turn/steer'].includes(request.method))).toEqual([]); + expect(control.markers.filter(marker => marker.kind === 'thread')).toEqual([ + { kind: 'thread', payload: { threadId: 'thread-existing' } }, + ]); + expect(control.states).toHaveLength(1); + expect(control.activities).toEqual([]); + expect(control.finals).toEqual([]); + expect(control.markers.some(marker => 'usage' in marker.payload || 'tokenUsage' in marker.payload)).toBe(false); + expect(harness.child.exitCode).toBeNull(); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it.each([ + { behavior: 'resume-not-found', threadId: 'thread-existing', error: 'not found', exitCode: 1 }, + { behavior: 'resume-different-thread', threadId: 'thread-existing', error: 'Strict resume expected thread thread-existing, received thread-unexpected', exitCode: 1 }, + { behavior: 'success', threadId: undefined, error: '--strict-resume requires --thread-id', exitCode: 2 }, + ])('strict resume fails closed for $behavior / $threadId', async ({ behavior, threadId, error, exitCode }) => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-strict-resume-rejection-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir); + await control.listen(); + const harness = startRunner( + fakeCodex, dir, logPath, '0.153.4', behavior, control.bootstrap.path, + { threadId, extraArgs: ['--strict-resume'] }, + ); + try { + const actualExitCode = await new Promise(resolvePromise => harness.child.once('exit', resolvePromise)); + const requests = readRequests(logPath); + expect(actualExitCode).toBe(exitCode); + expect(requests.filter(request => request.method === 'thread/resume')).toHaveLength(threadId ? 1 : 0); + expect(requests.filter(request => ['thread/start', 'turn/start'].includes(request.method))).toEqual([]); + expect(control.states).toEqual([]); + expect(control.finals).toEqual([]); + expect(harness.stdout).not.toContain('Codex App connected.'); + expect(harness.stderr).toContain(error); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('keeps the normal missing-thread recovery when strict resume is not requested', async () => { + const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-normal-resume-')); + const fakeCodex = join(dir, 'fake-codex'); + const logPath = join(dir, 'requests.jsonl'); + copyFileSync(FAKE_SERVER_FIXTURE, fakeCodex); + chmodSync(fakeCodex, 0o755); + const control = new ControlCollector(dir); + await control.listen(); + const harness = startRunner( + fakeCodex, dir, logPath, '0.153.4', 'resume-not-found', control.bootstrap.path, + { threadId: 'thread-missing' }, + ); + try { + await waitFor(harness, () => control.states.some(state => state.busy === false)); + const requests = readRequests(logPath); + expect(requests.filter(request => request.method === 'thread/resume')).toHaveLength(1); + expect(requests.filter(request => request.method === 'thread/start')).toHaveLength(1); + expect(requests.filter(request => request.method === 'turn/start')).toEqual([]); + expect(control.markers).toContainEqual({ kind: 'thread', payload: { threadId: 'thread-fake' } }); + } finally { + await stopChild(harness.child); + await control.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it('does not turn an ambiguous resume timeout into a fresh thread', async () => { const dir = mkdtempSync(join(tmpdir(), 'botmux-codex-runner-resume-timeout-')); const fakeCodex = join(dir, 'fake-codex'); diff --git a/test/codex-quiet-resume.e2e.ts b/test/codex-quiet-resume.e2e.ts new file mode 100644 index 000000000..6cf610f5b --- /dev/null +++ b/test/codex-quiet-resume.e2e.ts @@ -0,0 +1,202 @@ +/** Real Codex, local-only Responses provider, isolated home; no account or model + * credentials are copied. Opt in with BOTMUX_TEST_REAL_CODEX_BIN=/path/to/codex. + * The seed turn uses a deterministic localhost response; both subsequent + * resumes must stay idle without making another inference request. */ +import { spawn, execFileSync, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import { createServer } from 'node:http'; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import * as pty from 'node-pty'; +import { describe, expect, it } from 'vitest'; +import { createCodexAdapter } from '../src/adapters/cli/codex.js'; + +const codexBin = process.env.BOTMUX_TEST_REAL_CODEX_BIN; +const observeMs = 2_100; + +async function until(predicate: () => boolean, detail: () => string, timeoutMs = 20_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() >= deadline) throw new Error(`Timed out: ${detail()}`); + await new Promise(resolve => setTimeout(resolve, 20)); + } +} + +function startAppServer(bin: string, cwd: string, env: Record) { + const child = spawn(bin, ['app-server', '--stdio'], { cwd, env, stdio: ['pipe', 'pipe', 'pipe'] }); + let stderr = ''; + let buffer = ''; + let sequence = 0; + const notifications: any[] = []; + const pending = new Map(); + child.stderr.on('data', chunk => { stderr += String(chunk); }); + child.stdout.on('data', chunk => { + buffer += String(chunk); + let newline: number; + while ((newline = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + if (!line.trim()) continue; + const message = JSON.parse(line); + const entry = pending.get(message.id); + if (entry) { + pending.delete(message.id); + if (message.error) entry.reject(new Error(JSON.stringify(message.error))); + else entry.resolve(message.result); + } else notifications.push(message); + } + }); + child.once('exit', code => { + for (const entry of pending.values()) entry.reject(new Error(`Codex exited ${code}: ${stderr}`)); + pending.clear(); + }); + return { + child, notifications, + get stderr() { return stderr; }, + request(method: string, params: Record): Promise { + const id = ++sequence; + const result = new Promise((resolve, reject) => pending.set(id, { resolve, reject })); + child.stdin.write(`${JSON.stringify({ id, method, params })}\n`); + return result; + }, + notify(method: string) { child.stdin.write(`${JSON.stringify({ method })}\n`); }, + }; +} + +async function stop(child: ChildProcessWithoutNullStreams): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + await new Promise(resolve => { + const timer = setTimeout(() => child.kill('SIGKILL'), 1_000); + child.once('exit', () => { clearTimeout(timer); resolve(); }); + child.kill('SIGTERM'); + }); +} + +function usageRecords(codexHome: string): unknown[] { + const result: unknown[] = []; + function visit(dir: string) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) visit(path); + else if (entry.name.endsWith('.jsonl')) { + for (const line of readFileSync(path, 'utf8').trim().split('\n')) { + if (!line) continue; + const record = JSON.parse(line); + if (record.type === 'event_msg' && record.payload?.type === 'token_count') result.push(record.payload.info); + } + } + } + } + visit(join(codexHome, 'sessions')); + return result; +} + +describe.skipIf(!codexBin)('real Codex quiet maintenance resume', () => { + it('restores the original history through app-server and TUI without extra inference or usage', async () => { + const directory = mkdtempSync(join(tmpdir(), 'botmux-real-codex-quiet-resume-')); + const codexHome = join(directory, '.codex'); + const workspace = join(directory, 'workspace'); + mkdirSync(codexHome); + mkdirSync(workspace); + const requests: Array<{ method: string; path: string }> = []; + const server = createServer(async (req, res) => { + requests.push({ method: req.method ?? '', path: req.url ?? '' }); + for await (const _chunk of req) { /* consume local request without storing prompts */ } + if (req.url === '/v1/models') { + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ data: [{ id: 'quiet-resume-test', object: 'model', owned_by: 'local-test' }] })); + return; + } + if (req.url !== '/v1/responses') { res.writeHead(404).end(); return; } + res.writeHead(200, { 'content-type': 'text/event-stream' }); + const message = { id: 'msg_local', type: 'message', role: 'assistant', status: 'completed', content: [{ type: 'output_text', text: 'Local seed answer.', annotations: [] }] }; + const response = { id: 'resp_local', object: 'response', created_at: 1, model: 'quiet-resume-test', status: 'completed', output: [message], usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15, input_tokens_details: { cached_tokens: 0 }, output_tokens_details: { reasoning_tokens: 0 } } }; + const events = [ + { type: 'response.created', response: { ...response, status: 'in_progress', output: [] } }, + { type: 'response.output_item.added', output_index: 0, item: { ...message, status: 'in_progress', content: [] } }, + { type: 'response.content_part.added', item_id: message.id, output_index: 0, content_index: 0, part: { type: 'output_text', text: '', annotations: [] } }, + { type: 'response.output_text.delta', item_id: message.id, output_index: 0, content_index: 0, delta: 'Local seed answer.' }, + { type: 'response.output_text.done', item_id: message.id, output_index: 0, content_index: 0, text: 'Local seed answer.' }, + { type: 'response.output_item.done', output_index: 0, item: message }, + { type: 'response.completed', response }, + ]; + for (const [sequence_number, event] of events.entries()) res.write(`event: ${event.type}\ndata: ${JSON.stringify({ ...event, sequence_number })}\n\n`); + res.end(); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Expected localhost TCP provider'); + writeFileSync(join(codexHome, 'config.toml'), `model = "quiet-resume-test" +model_provider = "local_test" +check_for_update_on_startup = false +[analytics] +enabled = false +[model_providers.local_test] +name = "Isolated local test" +base_url = "http://127.0.0.1:${address.port}/v1" +wire_api = "responses" +requires_openai_auth = false +env_key = "BOTMUX_TEST_LOCAL_API_KEY" +`); + const env = { PATH: process.env.PATH ?? '/usr/bin:/bin', HOME: directory, CODEX_HOME: codexHome, TERM: 'xterm-256color', BOTMUX_TEST_LOCAL_API_KEY: 'local-fixture-only' }; + const processes: ChildProcessWithoutNullStreams[] = []; + let terminal: pty.IPty | undefined; + let terminalExited = false; + try { + const seed = startAppServer(codexBin!, workspace, env); + processes.push(seed.child); + await seed.request('initialize', { clientInfo: { name: 'botmux-quiet-resume-test', version: '1.0.0' }, capabilities: { experimentalApi: true } }); + seed.notify('initialized'); + const started = await seed.request('thread/start', { cwd: workspace, approvalPolicy: 'never', sandbox: 'danger-full-access', persistExtendedHistory: true }); + const threadId = started.thread.id; + await seed.request('turn/start', { threadId, input: [{ type: 'text', text: 'Create local seed history.', text_elements: [] }] }); + await until(() => seed.notifications.some(event => event.method === 'turn/completed'), () => JSON.stringify({ notifications: seed.notifications, stderr: seed.stderr })); + expect(seed.notifications.find(event => event.method === 'turn/completed')?.params.turn.status).toBe('completed'); + expect(seed.notifications.some(event => event.method === 'thread/tokenUsage/updated')).toBe(true); + const seedUsage = seed.notifications.filter(event => event.method === 'thread/tokenUsage/updated').at(-1).params.tokenUsage.total; + await stop(seed.child); + const usageBefore = usageRecords(codexHome); + expect(usageBefore.length).toBeGreaterThan(0); + const requestCountBefore = requests.length; + expect(requests.filter(request => request.path === '/v1/responses')).toHaveLength(1); + + const resumed = startAppServer(codexBin!, workspace, env); + processes.push(resumed.child); + await resumed.request('initialize', { clientInfo: { name: 'botmux-quiet-resume-test', version: '1.0.0' }, capabilities: { experimentalApi: true } }); + resumed.notify('initialized'); + const restored = await resumed.request('thread/resume', { threadId, cwd: workspace, approvalPolicy: 'never', sandbox: 'danger-full-access', persistExtendedHistory: true }); + expect(restored.thread.id).toBe(threadId); + expect(restored.thread.turns.length).toBeGreaterThan(0); + await new Promise(resolve => setTimeout(resolve, observeMs)); + expect(requests).toHaveLength(requestCountBefore); + // Codex replays the persisted usage snapshot while resuming. It is not + // new consumption: every replay must equal the completed seed total. + const replayedUsage = resumed.notifications.filter(event => event.method === 'thread/tokenUsage/updated'); + for (const event of replayedUsage) expect(event.params.tokenUsage.total).toEqual(seedUsage); + expect(usageRecords(codexHome)).toEqual(usageBefore); + await stop(resumed.child); + + const args = createCodexAdapter(codexBin).buildArgs({ sessionId: 'quiet-resume-local-test', resume: true, resumeSessionId: threadId, quietResume: true, bypassHookTrust: true }); + let screen = ''; + terminal = pty.spawn(codexBin!, args, { name: 'xterm-256color', cols: 120, rows: 40, cwd: workspace, env }); + terminal.onData(data => { screen += data; }); + terminal.onExit(() => { terminalExited = true; }); + await until(() => screen.includes('quiet-resume-test') && screen.includes('›'), () => screen); + await new Promise(resolve => setTimeout(resolve, observeMs)); + expect(terminalExited).toBe(false); + expect(requests).toHaveLength(requestCountBefore); + expect(usageRecords(codexHome)).toEqual(usageBefore); + const report = { binary: codexBin, version: execFileSync(codexBin!, ['--version'], { env, encoding: 'utf8' }).trim(), threadId, provider: 'localhost fixture only', seedInferenceRequests: 1, seedUsage, replayedHistoricalUsageEvents: replayedUsage.length, appServerObservationMs: observeMs, tuiObservationMs: observeMs, additionalInferenceRequests: requests.length - requestCountBefore, additionalUsageRecords: usageRecords(codexHome).length - usageBefore.length, tuiArgs: args }; + if (process.env.BOTMUX_TEST_REAL_CODEX_REPORT) writeFileSync(process.env.BOTMUX_TEST_REAL_CODEX_REPORT, JSON.stringify(report, null, 2)); + console.log(JSON.stringify(report)); + } finally { + if (terminal && !terminalExited) { + terminal.kill(); + await until(() => terminalExited, () => 'PTY did not exit', 3_000); + } + await Promise.all(processes.map(stop)); + await new Promise(resolve => server.close(() => resolve())); + rmSync(directory, { recursive: true, force: true }); + } + }, 60_000); +}); diff --git a/test/codex-session-upgrade.test.ts b/test/codex-session-upgrade.test.ts new file mode 100644 index 000000000..1288a06c2 --- /dev/null +++ b/test/codex-session-upgrade.test.ts @@ -0,0 +1,258 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + CodexSessionUpgradeMonitor, + hasCodexAutonomousGoal, + isCodexProcess, + parseUpgradeProcessTable, + probeCodexExecutable, + upgradeProcessTree, + upgradeRequired, + type CodexExecutable, + type CodexProcess, +} from '../src/services/codex-session-upgrade.js'; + +const scratch: string[] = []; + +afterEach(() => { + for (const path of scratch.splice(0)) rmSync(path, { recursive: true, force: true }); +}); + +function temporaryDirectory(): string { + const path = mkdtempSync(join(tmpdir(), 'botmux-codex-upgrade-')); + scratch.push(path); + return path; +} + +function executable(version = '0.153.4'): CodexExecutable { + return { path: '/runtime/codex', version, fingerprint: `installed:${version}` }; +} + +function running(version = '0.146.0', pid = 100): CodexProcess { + return { ...executable(version), fingerprint: `running:${version}`, pid, started: 'process-birth' }; +} + +function snapshot() { + return { target: executable(), running: [running()] }; +} + +function monitorDependencies() { + return { + enabled: vi.fn(() => true), + check: vi.fn(async () => snapshot()), + blocked: vi.fn<() => string | undefined>(() => undefined), + upgrade: vi.fn(async (_target: CodexExecutable, _running: CodexProcess[]) => {}), + report: vi.fn(), + }; +} + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +describe('Codex session upgrade version boundary', () => { + it('upgrades an older running release when the installed version is higher', () => { + expect(upgradeRequired(executable(), [running()])).toBe(true); + expect(upgradeRequired(executable(), [running('0.153.4'), running('0.146.0', 101)])).toBe(true); + }); + + it('does not replace equal versions, downgrade, or infer a stale process from an empty list', () => { + expect(upgradeRequired(executable(), [])).toBe(false); + expect(upgradeRequired(executable(), [running('0.153.4')])).toBe(false); + expect(upgradeRequired(executable(), [running('0.154.0')])).toBe(false); + expect(upgradeRequired(executable('0.153.4-beta.1'), [running('0.153.4')])).toBe(false); + }); + + it('does not downgrade a newer member of a mixed-version session', () => { + expect(upgradeRequired(executable(), [running(), running('0.154.0', 101)])).toBe(false); + }); +}); + +describe('Codex session upgrade monitor', () => { + it('does no inspection while disabled and observes enabling on a later tick', async () => { + const deps = monitorDependencies(); + deps.enabled.mockReturnValue(false); + const monitor = new CodexSessionUpgradeMonitor(deps); + await monitor.tick(); + expect(deps.check).not.toHaveBeenCalled(); + deps.enabled.mockReturnValue(true); + await monitor.tick(); + expect(deps.upgrade).toHaveBeenCalledWith(executable(), [running()]); + expect(deps.report.mock.calls).toEqual([['upgrading', '0.153.4'], ['current', '0.153.4']]); + }); + + it('respects a hot disable while an asynchronous inspection is in flight', async () => { + const deps = monitorDependencies(); + const inspection = deferred>(); + deps.check.mockImplementationOnce(() => inspection.promise); + const monitor = new CodexSessionUpgradeMonitor(deps); + const tick = monitor.tick(); + deps.enabled.mockReturnValue(false); + inspection.resolve(snapshot()); + await tick; + expect(deps.blocked).not.toHaveBeenCalled(); + expect(deps.upgrade).not.toHaveBeenCalled(); + expect(deps.report).not.toHaveBeenCalled(); + }); + + it('serializes concurrent ticks through both inspection and replacement', async () => { + const deps = monitorDependencies(); + const inspection = deferred>(); + const replacement = deferred(); + deps.check.mockImplementationOnce(() => inspection.promise); + deps.upgrade.mockImplementationOnce(() => replacement.promise); + const monitor = new CodexSessionUpgradeMonitor(deps); + const first = monitor.tick(); + await monitor.tick(); + expect(deps.check).toHaveBeenCalledTimes(1); + inspection.resolve(snapshot()); + await Promise.resolve(); + expect(deps.upgrade).toHaveBeenCalledTimes(1); + await monitor.tick(); + expect(deps.check).toHaveBeenCalledTimes(1); + replacement.resolve(); + await first; + await monitor.tick(); + expect(deps.check).toHaveBeenCalledTimes(2); + }); + + it('waits for a blocked session without repeatedly reporting the same reason', async () => { + const deps = monitorDependencies(); + deps.blocked.mockReturnValue('active turn'); + const monitor = new CodexSessionUpgradeMonitor(deps); + await monitor.tick(); + await monitor.tick(); + expect(deps.upgrade).not.toHaveBeenCalled(); + expect(deps.report.mock.calls).toEqual([['waiting', 'active turn']]); + deps.blocked.mockReturnValue(undefined); + await monitor.tick(); + expect(deps.upgrade).toHaveBeenCalledTimes(1); + expect(deps.report.mock.calls).toEqual([ + ['waiting', 'active turn'], ['upgrading', '0.153.4'], ['current', '0.153.4'], + ]); + }); + + it('does not upgrade after an inspection error and can inspect again on a later tick', async () => { + const deps = monitorDependencies(); + deps.check.mockRejectedValueOnce(new Error('process identity unavailable')); + const monitor = new CodexSessionUpgradeMonitor(deps); + await monitor.tick(); + expect(deps.upgrade).not.toHaveBeenCalled(); + expect(deps.report.mock.calls).toEqual([['failed', 'process identity unavailable']]); + await monitor.tick(); + expect(deps.check).toHaveBeenCalledTimes(2); + expect(deps.upgrade).toHaveBeenCalledTimes(1); + expect(deps.report).toHaveBeenLastCalledWith('current', '0.153.4'); + }); + + it('reports a failed replacement without publishing current and releases the in-flight gate', async () => { + const deps = monitorDependencies(); + deps.upgrade.mockRejectedValueOnce(new Error('old process is still alive')); + const monitor = new CodexSessionUpgradeMonitor(deps); + await monitor.tick(); + expect(deps.report.mock.calls).toEqual([ + ['upgrading', '0.153.4'], ['failed', 'old process is still alive'], + ]); + await monitor.tick(); + expect(deps.upgrade).toHaveBeenCalledTimes(2); + expect(deps.report).toHaveBeenLastCalledWith('current', '0.153.4'); + }); + + it('does not run replacement for an unchanged or newer running version', async () => { + const deps = monitorDependencies(); + deps.check.mockResolvedValueOnce({ target: executable(), running: [running('0.153.4')] }); + deps.check.mockResolvedValueOnce({ target: executable(), running: [running('0.154.0')] }); + const monitor = new CodexSessionUpgradeMonitor(deps); + await monitor.tick(); + await monitor.tick(); + expect(deps.blocked).not.toHaveBeenCalled(); + expect(deps.upgrade).not.toHaveBeenCalled(); + }); +}); + +describe('Codex autonomous goal detection', () => { + function rollout(entries: unknown[]): string { + const path = join(temporaryDirectory(), 'rollout.jsonl'); + writeFileSync(path, entries.map(entry => JSON.stringify(entry)).join('\n') + '\n'); + return path; + } + + it.each([ + { type: 'event_msg', payload: { type: 'goal_updated', status: 'active' } }, + { type: 'response_item', payload: { type: 'function_call', name: 'functions.create_goal' } }, + { type: 'response_item', payload: { type: 'function_call', name: 'goal/set' } }, + ])('recognizes native goal evidence: $type $payload.type', async entry => { + expect(await hasCodexAutonomousGoal(rollout([entry]))).toBe(true); + }); + + it('does not treat ordinary message text mentioning goals as autonomous work', async () => { + const path = rollout([ + { type: 'session_meta', payload: { id: 'session' } }, + { type: 'event_msg', payload: { type: 'agent_message', message: 'Discuss create_goal and goal/set.' } }, + { type: 'response_item', payload: { type: 'function_call', name: 'exec_command', arguments: '{"cmd":"echo create_goal"}' } }, + ]); + expect(await hasCodexAutonomousGoal(path)).toBe(false); + }); + + it('rejects malformed JSONL instead of treating the session as safely idle', async () => { + const path = rollout([{ type: 'session_meta', payload: { id: 'session' } }]); + writeFileSync(path, '{"type":"event_msg","payload":'); + await expect(hasCodexAutonomousGoal(path)).rejects.toThrow(); + }); + + it('rejects a missing rollout instead of inferring that no goal exists', async () => { + await expect(hasCodexAutonomousGoal(join(temporaryDirectory(), 'missing.jsonl'))).rejects.toThrow(); + }); +}); + +describe('Codex process ownership scope', () => { + it('parses process paths with spaces and ignores headers or malformed rows', () => { + expect(parseUpgradeProcessTable('PID PPID COMMAND\n 12 1 /Applications/Codex App/codex\ninvalid\n13 12 /bin/sh\n')).toEqual([ + { pid: 12, ppid: 1, command: '/Applications/Codex App/codex' }, + { pid: 13, ppid: 12, command: '/bin/sh' }, + ]); + }); + + it('includes descendants despite table ordering and excludes sibling or externally owned app servers', () => { + const rows = parseUpgradeProcessTable([ + '13 12 /runtime/codex', + '11 10 /bin/sh', + '22 20 /runtime/codex', + '12 11 /usr/bin/node', + '10 1 /runtime/botmux-worker', + '20 1 /Applications/Codex/codex', + '14 10 /runtime/claude', + ].join('\n')); + expect(upgradeProcessTree(rows, [10]).map(row => row.pid)).toEqual([13, 11, 12, 10, 14]); + expect(upgradeProcessTree(rows, []).map(row => row.pid)).toEqual([]); + expect(upgradeProcessTree(rows, [10]).filter(isCodexProcess).map(row => row.pid)).toEqual([13]); + }); +}); + +describe.skipIf(process.platform === 'win32')('Codex executable version probe', () => { + function fixture(output: string): string { + const path = join(temporaryDirectory(), 'codex-fixture'); + // This is a POSIX executable fixture, not a TypeScript child process. + writeFileSync(path, '#!/bin/sh\n[ "$#" -eq 1 ] && [ "$1" = "--version" ] || exit 19\n' + + `printf '%s\\n' '${output}'\n`, { mode: 0o755 }); + return path; + } + + it('only requests --version and resolves the executable behind a launcher symlink', async () => { + const target = fixture('codex-cli 0.153.4'); + const alias = join(temporaryDirectory(), 'codex'); + symlinkSync(target, alias); + const result = await probeCodexExecutable(alias); + expect(result.path).toBe(realpathSync(target)); + expect(result.version).toBe('0.153.4'); + expect(result.fingerprint).toContain(realpathSync(target)); + }); + + it.each(['node v22.0.0', 'VendorCodex 0.153.4', '0.153.4'])('rejects non-Codex version output: %s', async output => { + await expect(probeCodexExecutable(fixture(output))).rejects.toThrow('did not identify itself as Codex'); + }); +}); diff --git a/test/codex-upgrade-helpers.test.ts b/test/codex-upgrade-helpers.test.ts new file mode 100644 index 000000000..1f670dfbc --- /dev/null +++ b/test/codex-upgrade-helpers.test.ts @@ -0,0 +1,251 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; + +const probes = vi.hoisted(() => ({ + cmdline: vi.fn<(pid: number) => string[]>(), + identity: vi.fn<(pid: number) => string | undefined>(), + procExecutables: new Map(), +})); + +vi.mock('../src/core/session-discovery.js', () => ({ readCmdline: probes.cmdline })); +vi.mock('../src/utils/process-identity.js', () => ({ readProcessStartIdentity: probes.identity })); +vi.mock('node:fs', async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + realpathSync: vi.fn((path: string) => actual.realpathSync(probes.procExecutables.get(path) ?? path)), + }; +}); + +import { isRestartableCodexHelper, type CodexProcess, type ProcessRow } from '../src/services/codex-session-upgrade.js'; + +const platformDescriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; +let directory: string; +let running: CodexProcess; +let gateway: string; +let cliEntry: string; +let node: string; +let codeMode: string; + +function fixture(path: string, text = ''): string { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, text); + return path; +} + +function helper(executable: string, argv: string[], ppid = 100): ProcessRow { + const row = { pid: 200, ppid, command: executable }; + probes.cmdline.mockReturnValue(argv); + probes.procExecutables.set('/proc/200/exe', executable); + return row; +} + +function eligible(row: ProcessRow, tree: ProcessRow[] = [row]): boolean { + return isRestartableCodexHelper(row, tree, [running], gateway); +} + +function npmPlatformBinary(packageName: string, nestedDirectory = 'node_modules') { + const owner = join(directory, 'npm-global', 'lib', 'node_modules', 'botmux'); + const platformRoot = join(owner, nestedDirectory, packageName); + const binary = fixture(join(platformRoot, 'botmux')); + const platformPackage = { name: packageName, version: '3.18.11' }; + const ownerPackage = { + name: 'botmux', version: '3.18.11', optionalDependencies: { [packageName]: '3.18.11' }, + }; + const platformMetadata = fixture(join(platformRoot, 'package.json'), JSON.stringify(platformPackage)); + const ownerMetadata = fixture(join(owner, 'package.json'), JSON.stringify(ownerPackage)); + // The live leaf still executes its npm platform binary after the gateway + // switches to a newly installed standalone release. + const current = fixture(join(directory, 'current-release', 'botmux')); + writeFileSync(gateway, `#!/bin/sh\nexec "${current}" "$@"\n`); + return { binary, platformMetadata, platformPackage, ownerMetadata, ownerPackage }; +} + +beforeEach(() => { + probes.cmdline.mockReset(); + probes.identity.mockReset().mockReturnValue('stable-start'); + probes.procExecutables.clear(); + vi.mocked(realpathSync).mockClear(); + directory = realpathSync(mkdtempSync(join(tmpdir(), 'botmux-codex-helper-'))); + running = { + pid: 100, started: 'codex-start', version: '0.146.0', fingerprint: 'running', + path: fixture(join(directory, 'releases', '0.146.0', 'bin', 'codex')), + }; + codeMode = fixture(join(dirname(running.path), 'codex-code-mode-host')); + node = fixture(join(directory, 'node', 'bin', 'node')); + cliEntry = fixture(join(directory, 'botmux', 'dist', 'cli.js')); + gateway = fixture(join(directory, 'bin', 'botmux'), `#!/bin/sh\nexec node "${cliEntry}" "$@"\n`); +}); + +afterEach(() => { + Object.defineProperty(process, 'platform', platformDescriptor); + rmSync(directory, { recursive: true, force: true }); +}); + +describe.each(['darwin', 'linux'] as const)('restartable Codex leaf helpers on %s', platform => { + beforeEach(() => Object.defineProperty(process, 'platform', { ...platformDescriptor, value: platform })); + + it('allows a code-mode leaf from its direct Codex parent release directory', () => { + const row = helper(codeMode, [codeMode]); + if (platform === 'linux') row.command = 'codex-code-mod'; // ps comm can be truncated; /proc is authoritative. + expect(eligible(row)).toBe(true); + expect(realpathSync).toHaveBeenCalledWith(platform === 'linux' ? '/proc/200/exe' : codeMode); + expect(probes.identity.mock.calls).toEqual([[200], [200]]); + }); + + it('rejects a code-mode executable from another release or an unrelated parent', () => { + const other = fixture(join(directory, 'releases', '0.153.4', 'bin', 'codex-code-mode-host')); + expect(eligible(helper(other, [other]))).toBe(false); + expect(eligible(helper(codeMode, [codeMode], 101))).toBe(false); + }); + + it.each([false, true])('rejects a known helper with children (grandchildren=%s)', grandchildren => { + const row = helper(codeMode, [codeMode]); + const child = { pid: 201, ppid: row.pid, command: node }; + const tree = [row, child, ...(grandchildren ? [{ pid: 202, ppid: child.pid, command: node }] : [])]; + expect(eligible(row, tree)).toBe(false); + expect(probes.cmdline).not.toHaveBeenCalled(); + }); + + it('allows the exact configured gateway executable with only mcp serve arguments', () => { + expect(eligible(helper(gateway, [gateway, 'mcp', 'serve']))).toBe(true); + const unrelated = fixture(join(directory, 'user', 'botmux')); + expect(eligible(helper(unrelated, [unrelated, 'mcp', 'serve']))).toBe(false); + }); + + it('allows Node only for the exact CLI entry from the configured Botmux wrapper', () => { + expect(eligible(helper(node, [node, cliEntry, 'mcp', 'serve']))).toBe(true); + const otherEntry = fixture(join(directory, 'other-checkout', 'dist', 'cli.js')); + expect(eligible(helper(node, [node, otherEntry, 'mcp', 'serve']))).toBe(false); + }); + + it('allows the exact standalone binary target from the configured two-line wrapper', () => { + const binary = fixture(join(directory, 'binary release', 'botmux')); + writeFileSync(gateway, `#!/bin/sh\nexec "${binary}" "$@"\n`); + expect(eligible(helper(binary, [binary, 'mcp', 'serve']))).toBe(true); + const oldBinary = fixture(join(directory, 'old-release', 'botmux')); + expect(eligible(helper(oldBinary, [oldBinary, 'mcp', 'serve']))).toBe(false); + }); + + it.each(['x64', 'arm64'])('allows an old installed npm %s gateway leaf after the launcher changes release', arch => { + const { binary } = npmPlatformBinary(`botmux-${platform}-${arch}`); + const row = helper(binary, [binary, 'mcp', 'serve']); + if (platform === 'linux') row.command = 'botmux'; + expect(eligible(row)).toBe(true); + expect(realpathSync).toHaveBeenCalledWith(platform === 'linux' ? '/proc/200/exe' : binary); + }); + + it.each(['x64', 'arm64'])('accepts Linux %s musl packages and rejects a fabricated Darwin musl package', arch => { + const { binary } = npmPlatformBinary(`botmux-${platform}-${arch}-musl`); + expect(eligible(helper(binary, [binary, 'mcp', 'serve']))).toBe(platform === 'linux'); + }); + + it('requires the old installed npm helper to be a direct Codex child with no children', () => { + const { binary } = npmPlatformBinary(`botmux-${platform}-arm64`); + expect(eligible(helper(binary, [binary, 'mcp', 'serve'], 101))).toBe(false); + const row = helper(binary, [binary, 'mcp', 'serve']); + expect(eligible(row, [row, { pid: 201, ppid: row.pid, command: node }])).toBe(false); + }); + + it.each(['extra', 'missing', 'different-command'])('rejects an old npm binary with %s argv', kind => { + const { binary } = npmPlatformBinary(`botmux-${platform}-arm64`); + const argv = kind === 'extra' ? [binary, 'mcp', 'serve', '--custom'] + : kind === 'missing' ? [binary, 'mcp'] : [binary, 'daemon', 'start']; + expect(eligible(helper(binary, argv))).toBe(false); + }); + + it.each(['owner', 'platform'])('rejects missing %s package metadata', which => { + const installed = npmPlatformBinary(`botmux-${platform}-arm64`); + rmSync(which === 'owner' ? installed.ownerMetadata : installed.platformMetadata); + expect(eligible(helper(installed.binary, [installed.binary, 'mcp', 'serve']))).toBe(false); + }); + + it.each(['platform-name', 'owner-name', 'owner-version', 'dependency-version', 'dependency-range', 'missing-dependency', 'malformed-json'] as const)('rejects npm metadata mismatch: %s', kind => { + const installed = npmPlatformBinary(`botmux-${platform}-arm64`); + const { platformPackage, ownerPackage } = installed; + if (kind === 'platform-name') platformPackage.name = 'unrelated-platform-package'; + if (kind === 'owner-name') ownerPackage.name = 'unrelated-owner'; + if (kind === 'owner-version') ownerPackage.version = '3.18.12'; + if (kind === 'dependency-version') ownerPackage.optionalDependencies[platformPackage.name] = '3.18.12'; + if (kind === 'dependency-range') ownerPackage.optionalDependencies[platformPackage.name] = '^3.18.11'; + if (kind === 'missing-dependency') delete ownerPackage.optionalDependencies[platformPackage.name]; + writeFileSync(installed.platformMetadata, JSON.stringify(platformPackage)); + writeFileSync(installed.ownerMetadata, kind === 'malformed-json' ? '{' : JSON.stringify(ownerPackage)); + expect(eligible(helper(installed.binary, [installed.binary, 'mcp', 'serve']))).toBe(false); + }); + + it('rejects matching metadata placed outside the nested node_modules installation', () => { + const { binary } = npmPlatformBinary(`botmux-${platform}-arm64`, 'arbitrary-folder'); + expect(eligible(helper(binary, [binary, 'mcp', 'serve']))).toBe(false); + }); + + it.each(['renamed-package', 'renamed-binary'])('rejects forged npm installation names: %s', kind => { + const packageName = kind === 'renamed-package' ? `fake-botmux-${platform}-arm64` : `botmux-${platform}-arm64`; + const installed = npmPlatformBinary(packageName); + const binary = kind === 'renamed-binary' + ? fixture(join(dirname(installed.binary), 'other-command')) : installed.binary; + expect(eligible(helper(binary, [binary, 'mcp', 'serve']))).toBe(false); + }); + + it.skipIf(platform !== 'linux')('rejects a forged npm path in argv and ps when /proc identifies another executable', () => { + const { binary } = npmPlatformBinary('botmux-linux-x64'); + const row = helper(binary, [binary, 'mcp', 'serve']); + probes.procExecutables.set('/proc/200/exe', fixture(join(directory, 'user-command', 'botmux'))); + expect(eligible(row)).toBe(false); + expect(realpathSync).toHaveBeenCalledWith('/proc/200/exe'); + }); + + it.each(['extra-argument', 'extra-statement', 'prepended-statement', 'variable-command', 'command-substitution', 'backtick-substitution', 'escaped-path', 'relative-path', 'unquoted-command'] as const)('rejects a standalone wrapper with %s', kind => { + const binary = fixture(join(directory, 'binary', 'botmux')); + const wrappers: Record = { + 'extra-argument': `#!/bin/sh\nexec "${binary}" --extra "$@"\n`, + 'extra-statement': `#!/bin/sh\nexec "${binary}" "$@"; echo unwanted\n`, + 'prepended-statement': `#!/bin/sh\necho unwanted\nexec "${binary}" "$@"\n`, + 'variable-command': '#!/bin/sh\nexec "$BOTMUX_BIN" "$@"\n', + 'command-substitution': '#!/bin/sh\nexec "/tmp/$(printf botmux)" "$@"\n', + 'backtick-substitution': '#!/bin/sh\nexec "/tmp/`printf botmux`" "$@"\n', + 'escaped-path': `#!/bin/sh\nexec "${directory}/binary/\\botmux" "$@"\n`, + 'relative-path': '#!/bin/sh\nexec "./botmux" "$@"\n', + 'unquoted-command': `#!/bin/sh\nexec ${binary} "$@"\n`, + }; + writeFileSync(gateway, wrappers[kind]); + expect(eligible(helper(binary, [binary, 'mcp', 'serve']))).toBe(false); + }); + + it.each(['node', 'npm', 'user-mcp', 'repl', 'extra-arg', 'code-mode-arg'] as const)('rejects unknown %s processes', kind => { + const userMcp = fixture(join(directory, 'user', 'mcp.js')); + const npm = fixture(join(directory, 'node', 'bin', 'npm')); + const cases: Record = { + node: [node, [node]], + npm: [npm, [npm, 'exec', 'mcp', 'serve']], + 'user-mcp': [node, [node, userMcp, 'mcp', 'serve']], + repl: [node, [node, '--interactive']], + 'extra-arg': [node, [node, cliEntry, 'mcp', 'serve', '--custom']], + 'code-mode-arg': [codeMode, [codeMode, '--interactive']], + }; + const [executable, argv] = cases[kind]; + expect(eligible(helper(executable, argv))).toBe(false); + }); + + it('rejects a missing or changed PID identity', () => { + const row = helper(codeMode, [codeMode]); + probes.identity.mockReturnValueOnce(undefined); + expect(eligible(row)).toBe(false); + probes.identity.mockReturnValueOnce('old-start').mockReturnValueOnce('new-start'); + expect(eligible(row)).toBe(false); + probes.identity.mockReturnValueOnce('old-start').mockReturnValueOnce(undefined); + expect(eligible(row)).toBe(false); + }); + + it('rejects unreadable command lines, executables, and wrapper identities', () => { + expect(eligible(helper(codeMode, []))).toBe(false); + const gone = join(directory, 'gone-code-mode-host'); + expect(eligible(helper(gone, [gone]))).toBe(false); + writeFileSync(gateway, '#!/bin/sh\nnode arbitrary-script.js\n'); + expect(eligible(helper(node, [node, cliEntry, 'mcp', 'serve']))).toBe(false); + rmSync(gateway); + expect(eligible(helper(node, [node, cliEntry, 'mcp', 'serve']))).toBe(false); + }); +}); diff --git a/test/codex-upgrade-target.test.ts b/test/codex-upgrade-target.test.ts new file mode 100644 index 000000000..eac6139ae --- /dev/null +++ b/test/codex-upgrade-target.test.ts @@ -0,0 +1,137 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, unlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; + +const mocks = vi.hoisted(() => ({ homedir: vi.fn<() => string>(), resolve: vi.fn<(command: string) => string>() })); +vi.mock('node:os', async importOriginal => ({ ...await importOriginal(), homedir: mocks.homedir })); +vi.mock('../src/adapters/cli/registry.js', () => ({ resolveCommandReal: mocks.resolve })); + +import { resolveCodexUpgradeCommand } from '../src/services/codex-upgrade-target.js'; + +let directory: string; +let home: string; +let standalone: string; +let current: string; +let entry: string; +let oldNpm: string; + +function file(path: string): string { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, '#!/bin/sh\n', { mode: 0o755 }); + return path; +} + +function install(version = '0.153.4'): string { + const target = file(join(standalone, 'releases', version, 'bin', 'codex')); + symlinkSync(join('releases', version), current); + mkdirSync(dirname(entry), { recursive: true }); + symlinkSync(join(current, 'bin', 'codex'), entry); + return realpathSync(target); +} + +beforeEach(() => { + directory = realpathSync(mkdtempSync(join(tmpdir(), 'botmux-codex-upgrade-target-'))); + home = join(directory, 'home'); + mkdirSync(home); + standalone = join(home, '.codex', 'packages', 'standalone'); + current = join(standalone, 'current'); + entry = join(home, '.local', 'bin', 'codex'); + oldNpm = file(join(directory, 'npm', 'bin', 'codex')); + mocks.homedir.mockReset().mockReturnValue(home); + mocks.resolve.mockReset().mockImplementation(command => command === 'codex' ? oldNpm : realpathSync(command)); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + rmSync(directory, { recursive: true, force: true }); +}); + +describe('Codex automatic-upgrade installation selection', () => { + it.each([undefined, { source: 'official' }])('prefers managed current to the older npm PATH binary for runtime %j', cliRuntime => { + const managed = install(); + expect(resolveCodexUpgradeCommand({ cliRuntime })).toBe(managed); + expect(mocks.resolve).toHaveBeenCalledWith(managed); + expect(mocks.resolve).not.toHaveBeenCalledWith('codex'); + }); + + it('keeps an explicit override even when the official managed installation exists', () => { + install(); + const override = file(join(directory, 'custom', 'codex')); + expect(resolveCodexUpgradeCommand({ cliPathOverride: override })).toBe(override); + expect(mocks.resolve).toHaveBeenCalledWith(override); + expect(mocks.homedir).not.toHaveBeenCalled(); + }); + + it('keeps the configured runtime resolver, including an explicit configured path', () => { + install(); + const override = file(join(directory, 'configured', 'codex')); + expect(resolveCodexUpgradeCommand({ cliRuntime: { source: 'configured' } })).toBe(oldNpm); + expect(resolveCodexUpgradeCommand({ cliRuntime: { source: 'configured' }, cliPathOverride: override })).toBe(override); + expect(mocks.homedir).not.toHaveBeenCalled(); + }); + + it('uses the existing resolver when no managed current installation exists', () => { + expect(resolveCodexUpgradeCommand()).toBe(oldNpm); + expect(mocks.resolve).toHaveBeenCalledWith('codex'); + }); + + it('does not treat a broken current symlink as an absent installation', () => { + mkdirSync(standalone, { recursive: true }); + symlinkSync('releases/missing', current); + expect(() => resolveCodexUpgradeCommand()).toThrow('Cannot select the managed Codex upgrade target'); + expect(mocks.resolve).not.toHaveBeenCalled(); + }); + + it.each(['binary', 'entry'] as const)('rejects a managed installation missing its %s', missing => { + const managed = install(); + unlinkSync(missing === 'binary' ? managed : entry); + expect(() => resolveCodexUpgradeCommand()).toThrow('Cannot select the managed Codex upgrade target'); + expect(mocks.resolve).not.toHaveBeenCalled(); + }); + + it('rejects a user entry that still points at a different installation', () => { + install(); + unlinkSync(entry); + symlinkSync(oldNpm, entry); + expect(() => resolveCodexUpgradeCommand()).toThrow('does not point to managed current'); + expect(mocks.resolve).not.toHaveBeenCalled(); + }); + + it('rejects matching pointers that escape the canonical release directory', () => { + install(); + const outside = file(join(directory, 'outside', 'bin', 'codex')); + unlinkSync(current); + symlinkSync(dirname(dirname(outside)), current); + expect(() => resolveCodexUpgradeCommand()).toThrow('not a releases//bin/codex executable'); + expect(mocks.resolve).not.toHaveBeenCalled(); + }); + + it('rejects an extra nesting level inside releases', () => { + install(); + file(join(standalone, 'releases', '0.154.0', 'nested', 'bin', 'codex')); + unlinkSync(current); + symlinkSync('releases/0.154.0/nested', current); + expect(() => resolveCodexUpgradeCommand()).toThrow('not a releases//bin/codex executable'); + }); + + it('rereads current on every observation without caching the previous release', () => { + const first = install(); + expect(resolveCodexUpgradeCommand()).toBe(first); + for (const version of ['0.154.0', '0.155.0']) { + const next = file(join(standalone, 'releases', version, 'bin', 'codex')); + unlinkSync(current); + symlinkSync(join('releases', version), current); + expect(resolveCodexUpgradeCommand()).toBe(next); + } + }); + + it('canonicalizes a symlinked home and ignores a bot-specific CODEX_HOME', () => { + const managed = install(); + const alias = join(directory, 'home-alias'); + symlinkSync(home, alias); + mocks.homedir.mockReturnValue(alias); + vi.stubEnv('CODEX_HOME', join(directory, 'isolated-bot-codex')); + expect(resolveCodexUpgradeCommand()).toBe(managed); + }); +}); diff --git a/test/fixtures/fake-codex-app-server.mjs b/test/fixtures/fake-codex-app-server.mjs index 081432cde..7c6a2fa0a 100755 --- a/test/fixtures/fake-codex-app-server.mjs +++ b/test/fixtures/fake-codex-app-server.mjs @@ -383,6 +383,10 @@ function handle(request) { reject(request.id, -32600, `thread ${request.params.threadId} already has an active writer`); return; } + if (behavior === 'resume-different-thread') { + respond(request.id, { thread: { id: 'thread-unexpected' } }); + return; + } respond(request.id, { thread: { id: request.params.threadId } }); return; } diff --git a/test/settings-write-applier.test.ts b/test/settings-write-applier.test.ts index 60b5b278f..48c2a0010 100644 --- a/test/settings-write-applier.test.ts +++ b/test/settings-write-applier.test.ts @@ -27,6 +27,7 @@ function makeDeps(overrides: Partial = {}): SettingsWr chatBotDiscovery: true, herdrTraexPlugin: { enabled: false, source: '', ref: '', recommendedSource: '', recommendedRef: '' }, codexRpcInput: false, + autoUpgradeCodexSessions: true, codexNotifier: { enabled: false, targetBotAppId: null, @@ -177,6 +178,13 @@ describe('applySettingsWrite happy paths', () => { expect(deps.mergeDashboardConfig).toHaveBeenCalledWith({ noVisibleOutputHint: true }); }); + it.each([false, true])('writes autoUpgradeCodexSessions=%s through the dashboard segment', async (enabled) => { + const deps = makeDeps(); + const r = await applySettingsWrite({ autoUpgradeCodexSessions: enabled }, deps); + expect(r.ok).toBe(true); + expect(deps.mergeDashboardConfig).toHaveBeenCalledWith({ autoUpgradeCodexSessions: enabled }); + }); + it('writes bypassCodexHookTrust=false (the disable path — the whole point of a default-ON toggle)', async () => { const deps = makeDeps(); const r = await applySettingsWrite({ bypassCodexHookTrust: false }, deps); @@ -456,6 +464,13 @@ describe('applySettingsWrite — validation errors', () => { expect(deps.mergeDashboardConfig).not.toHaveBeenCalled(); }); + it.each(['true', 1, null])('rejects invalid autoUpgradeCodexSessions=%s without writing settings', async (value) => { + const deps = makeDeps(); + const r = await applySettingsWrite({ autoUpgradeCodexSessions: value }, deps); + expect(r).toEqual({ ok: false, error: 'invalid_autoUpgradeCodexSessions' }); + expect(deps.mergeDashboardConfig).not.toHaveBeenCalled(); + }); + it('rejects non-boolean bypassCodexHookTrust → invalid_bypassCodexHookTrust', async () => { const deps = makeDeps(); const r = await applySettingsWrite({ bypassCodexHookTrust: 'no' }, deps); diff --git a/test/worker-codex-session-upgrade.test.ts b/test/worker-codex-session-upgrade.test.ts new file mode 100644 index 000000000..5b0123536 --- /dev/null +++ b/test/worker-codex-session-upgrade.test.ts @@ -0,0 +1,483 @@ +/** Execute the worker's real swap and input-fence functions without importing + * its process entry point. Process, RPC, and terminal readiness are boundaries + * controlled by this harness; the ordering and failure gates are production code. */ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import ts from 'typescript'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { CodexExecutable, CodexProcess } from '../src/services/codex-session-upgrade.js'; + +const source = readFileSync(join(process.cwd(), 'src/worker.ts'), 'utf8'); +const parsed = ts.createSourceFile('worker.ts', source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS); + +function extracted(name: string): string { + const declaration = parsed.statements.find(statement => ts.isFunctionDeclaration(statement) + && statement.name?.text === name); + if (!declaration) throw new Error(`Missing worker function: ${name}`); + return ts.transpileModule(declaration.getText(parsed), { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.None }, + }).outputText; +} + +type Upgrade = (target: CodexExecutable, running: CodexProcess[]) => Promise; + +function evaluate(name: string, state: Record): T { + // A separate lexical scope per harness keeps mutable worker globals isolated. + // eslint-disable-next-line no-new-func + return new Function('state', `with (state) { ${extracted(name)}; return ${name}; }`)(state) as T; +} + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((yes, no) => { resolve = yes; reject = no; }); + return { promise, resolve, reject }; +} + +const target: CodexExecutable = { path: '/runtime/new/codex', version: '0.153.4', fingerprint: 'new-file' }; +const oldProcesses: CodexProcess[] = [{ path: '/runtime/old/codex', version: '0.146.0', fingerprint: 'old-file', pid: 101, started: 'old-birth' }]; +const replacement: CodexProcess[] = [{ ...target, pid: 201, started: 'new-birth' }]; +const threadId = '01a07a8a-134d-7bb1-bf6c-af19f14f3f4f'; + +function harness() { + const steps: string[] = []; + const delivered: unknown[] = []; + const state: Record = { + config: { autoUpgradeCodexSessions: true }, + lastInitConfig: { + cliId: 'codex', cliSessionId: threadId, sessionId: 'botmux-session', + workingDir: '/project', prompt: 'previous already executed prompt', + env: { CODEX_TEST_SETTING: 'original' }, + resume: false, forkSession: true, model: 'gpt-6-astra', reasoningEffort: 'high', + nativeSessionTitle: 'Old title', nativeSessionTitlePrompt: 'rename instructions', + }, + remoteThreadId: undefined, + codexAutoUpgrade: undefined, + cliRestartInProgress: false, + replacementSpawnInProgress: false, + rawInputRestartGate: false, + cliSpawnGeneration: 9, + intentionalRestartBackend: undefined, + awaitingFirstPrompt: false, + isPromptReady: true, + tmuxRestartTimer: undefined, + isFlushing: false, + injectionFlushing: false, + commandLineWritesPending: 0, + initialInputOwnershipPending: false, + pendingMessages: [], pendingRawInputs: [], pendingInjections: [], pendingAdoptMessages: [], + sessionRenameInFlight: () => false, + pendingSessionRename: undefined, + durableTurnInFlight: undefined, + tuiPromptBlocking: false, + hookReviewInputHold: false, + bareShellCheckInProgress: false, + ambiguousSubmissionRecoveryHold: undefined, + submitFailureChains: { size: () => 0 }, + codexAppTurnLiveness: { hasActiveTurn: () => false }, + codexAppCompletionAwaitingFinal: false, + codexAppTurnDispatchQueue: { size: () => 0 }, + codexAppRecoveredDispatches: [], + hasStructuredLifecycleBlock: () => false, + wsClients: new Set(), clientPtys: new Map(), + codexUpgradeInspectionBlock: undefined, + codexRunnerFreshness: 'current', + shouldHoldCodexRunnerInput: () => false, + cliAdapter: {}, + hasPendingInputForFlush: vi.fn(() => false), + log: vi.fn(), + send: vi.fn(), + sandboxEnabled: () => false, + setTimeout, + clearTimeout, + }; + const oldBackend = { destroySession: vi.fn(async () => { steps.push('destroy'); }) }; + state.backend = oldBackend; + state.killCli = vi.fn(() => { + steps.push('kill'); + state.backend = null; + state.remoteThreadId = undefined; + }); + state.waitForCodexExit = vi.fn(async () => { steps.push('exit confirmed'); }); + state.codexRpcEligible = vi.fn(() => false); + state.createCliAdapterSync = vi.fn(() => ({ id: 'codex' })); + state.prepareCliPluginGenerationAndGateway = vi.fn(async () => { steps.push('plugins'); }); + state.engageCodexRpc = vi.fn(async () => { + steps.push('rpc resume'); + state.remoteThreadId = threadId; + return 'resumed'; + }); + state.startScreenUpdates = vi.fn(); + state.startStuckDetector = vi.fn(); + state.spawnCli = vi.fn(async (_cfg: unknown) => { + steps.push('spawn'); + state.backend = { generation: 'replacement' }; + state.isPromptReady = false; + }); + state.inspectCodexUpgradeRuntime = vi.fn(async () => { + steps.push('inspect replacement'); + return replacement; + }); + state.findCodexRolloutSetByPid = vi.fn(() => { + steps.push('verify thread fd'); + return new Set([threadId]); + }); + state.codexUpgradeBlocked = evaluate<() => string | undefined>('codexUpgradeBlocked', state); + state.releaseRawInputRestartGate = evaluate<() => void>('releaseRawInputRestartGate', state); + state.flushPendingInjections = vi.fn(async () => { + steps.push('injections flush requested'); + expect(state.cliRestartInProgress).toBe(false); + expect(state.rawInputRestartGate).toBe(false); + }); + const realFlush = evaluate<() => Promise>('flushPending', state); + state.flushPending = vi.fn(async () => { + steps.push('flush requested'); + // Execute the production entrypoint to prove an armed fence returns before + // even consulting queued work. Actual CLI delivery is outside this harness. + await realFlush(); + if (!state.cliRestartInProgress) delivered.push(...state.pendingMessages.splice(0)); + }); + const upgrade = evaluate('autoUpgradeCodex', state); + const ready = () => { + state.isPromptReady = true; + state.awaitingFirstPrompt = false; + state.codexAutoUpgrade?.ready?.(); + }; + const run = () => upgrade(target, oldProcesses).then(() => undefined, error => error as Error); + return { state, oldBackend, steps, delivered, upgrade, ready, run, realFlush }; +} + +async function settleMicrotasks(): Promise { + // Boundaries in autoUpgradeCodex deliberately await several independent + // promises. No timers or real worker processes are needed to advance them. + for (let turn = 0; turn < 12; turn++) await Promise.resolve(); +} + +describe('worker ready-time Codex runtime version observation', () => { + function observationHarness() { + const state: Record = { + backend: { name: 'observed-backend' }, + cliSpawnGeneration: 3, + cliRestartInProgress: false, + codexAutoUpgrade: undefined, + inspectCodexUpgradeRuntime: vi.fn(async () => replacement), + send: vi.fn(), + }; + const observe = evaluate<() => Promise>('observeCodexRuntimeVersionOnReady', state); + return { state, observe }; + } + + it('publishes the observed version after all running Codex processes agree', async () => { + const h = observationHarness(); + h.state.inspectCodexUpgradeRuntime.mockResolvedValue([ + replacement[0], { ...replacement[0], pid: 202 }, + ]); + const done = h.observe(); + expect(h.state.send).not.toHaveBeenCalled(); + await done; + expect(h.state.send).toHaveBeenCalledExactlyOnceWith({ type: 'cli_runtime_version', version: '0.153.4' }); + }); + + it.each(['generation-changed', 'backend-replaced', 'backend-exited'] as const)('discards an asynchronous observation after %s', async change => { + const h = observationHarness(); + const inspected = deferred(); + h.state.inspectCodexUpgradeRuntime.mockImplementationOnce(() => inspected.promise); + const done = h.observe(); + expect(h.state.inspectCodexUpgradeRuntime).toHaveBeenCalledTimes(1); + if (change === 'generation-changed') h.state.cliSpawnGeneration++; + else if (change === 'backend-replaced') h.state.backend = { name: 'new-backend' }; + else h.state.backend = null; + inspected.resolve(replacement); + await done; + expect(h.state.send).not.toHaveBeenCalled(); + }); + + it.each(['cliRestartInProgress', 'codexAutoUpgrade'] as const)('does not publish past the %s validation fence', async fence => { + const h = observationHarness(); + const inspected = deferred(); + h.state.inspectCodexUpgradeRuntime.mockImplementationOnce(() => inspected.promise); + const done = h.observe(); + h.state[fence] = fence === 'cliRestartInProgress' ? true : { stage: 'restoring', threadId }; + inspected.resolve(replacement); + await done; + expect(h.state.send).not.toHaveBeenCalled(); + }); + + it('does not publish an observation that began without a live backend', async () => { + const h = observationHarness(); + h.state.backend = null; + await h.observe(); + expect(h.state.send).not.toHaveBeenCalled(); + }); + + it.each([ + { name: 'empty process list', processes: [] }, + { name: 'mixed running versions', processes: [replacement[0], oldProcesses[0]] }, + ])('rejects $name without reporting a version', async ({ processes }) => { + const h = observationHarness(); + h.state.inspectCodexUpgradeRuntime.mockResolvedValue(processes); + await expect(h.observe()).rejects.toThrow('Running Codex processes do not identify a single version'); + expect(h.state.send).not.toHaveBeenCalled(); + }); + + it('propagates an inspection failure for caller logging without publishing a version', async () => { + const h = observationHarness(); + const failure = new Error('runtime process identity is unavailable'); + h.state.inspectCodexUpgradeRuntime.mockRejectedValue(failure); + await expect(h.observe()).rejects.toBe(failure); + expect(h.state.send).not.toHaveBeenCalled(); + }); +}); + +describe('worker Codex session automatic upgrade', () => { + beforeEach(() => { vi.useFakeTimers(); }); + afterEach(() => { vi.useRealTimers(); }); + + it('leaves a configured CLI launcher in control of its runtime', async () => { + const h = harness(); + h.state.lastInitConfig.wrapperCli = 'custom-launcher codex'; + expect(await h.run()).toEqual(new Error('a configured CLI wrapper controls the runtime')); + expect(h.oldBackend.destroySession).not.toHaveBeenCalled(); + expect(h.state.spawnCli).not.toHaveBeenCalled(); + expect(h.state.cliRestartInProgress).toBe(false); + }); + + it('preserves a hook-review prompt and can upgrade after the hold is released', async () => { + const h = harness(); + h.state.hookReviewInputHold = true; + expect(h.state.codexUpgradeBlocked()).toBeDefined(); + expect(await h.run()).toBeInstanceOf(Error); + expect(h.oldBackend.destroySession).not.toHaveBeenCalled(); + expect(h.state.killCli).not.toHaveBeenCalled(); + expect(h.state.spawnCli).not.toHaveBeenCalled(); + expect(h.state.cliRestartInProgress).toBe(false); + expect(h.state.rawInputRestartGate).toBe(false); + + h.state.hookReviewInputHold = false; + const done = h.run(); + await settleMicrotasks(); + expect(h.state.spawnCli).toHaveBeenCalledTimes(1); + h.ready(); + expect(await done).toBeUndefined(); + }); + + it('confirms old CLI exit before spawning a replacement and resumes exactly the original thread', async () => { + const h = harness(); + h.state.lastInitConfig.cliPathOverride = '/configured/codex'; + const originalConfig = h.state.lastInitConfig; + const destroyed = deferred(); + const exited = deferred(); + h.oldBackend.destroySession.mockImplementationOnce(() => destroyed.promise); + h.state.waitForCodexExit.mockImplementationOnce(() => exited.promise); + const done = h.run(); + expect(h.state.cliRestartInProgress).toBe(true); + expect(h.state.rawInputRestartGate).toBe(true); + expect(h.state.cliSpawnGeneration).toBe(10); + expect(h.state.replacementSpawnInProgress).toBe(false); + expect(h.state.intentionalRestartBackend).toBe(h.oldBackend); + expect(h.state.killCli).not.toHaveBeenCalled(); + destroyed.resolve(); + await settleMicrotasks(); + expect(h.state.killCli).toHaveBeenCalledWith({ + preservePending: true, preservePolicyCapability: true, preserveInjections: true, + }); + expect(h.state.waitForCodexExit).toHaveBeenCalledWith(oldProcesses); + expect(h.state.spawnCli).not.toHaveBeenCalled(); + exited.resolve(); + await settleMicrotasks(); + expect(h.state.spawnCli).toHaveBeenCalledTimes(1); + expect(h.state.replacementSpawnInProgress).toBe(true); + expect(h.state.spawnCli).toHaveBeenCalledWith(expect.objectContaining({ + sessionId: 'botmux-session', workingDir: '/project', cliId: 'codex', + resume: true, cliSessionId: threadId, prompt: '', forkSession: false, + cliPathOverride: target.path, model: undefined, reasoningEffort: undefined, + nativeSessionTitle: undefined, nativeSessionTitlePrompt: undefined, + }), { pluginGenerationPrepared: false }); + expect(h.state.inspectCodexUpgradeRuntime).not.toHaveBeenCalled(); + h.ready(); + expect(await done).toBeUndefined(); + expect(h.state.replacementSpawnInProgress).toBe(false); + expect(h.state.lastInitConfig).toBe(originalConfig); + expect(h.state.lastInitConfig.cliPathOverride).toBe('/configured/codex'); + expect(h.state.send).toHaveBeenCalledWith({ type: 'cli_runtime_version', version: '0.153.4' }); + }); + + it('holds newly arriving inputs until readiness and runtime/thread fd verification both finish', async () => { + const h = harness(); + const inspected = deferred(); + h.state.inspectCodexUpgradeRuntime.mockImplementationOnce(() => inspected.promise); + const done = h.run(); + await settleMicrotasks(); + expect(h.state.replacementSpawnInProgress).toBe(true); + const pending = { turnId: 'new-turn', text: 'follow up arriving during restore' }; + h.state.pendingMessages.push(pending); + await h.realFlush(); + expect(h.state.hasPendingInputForFlush).not.toHaveBeenCalled(); + expect(h.state.pendingMessages).toEqual([pending]); + expect(h.state.flushPending).not.toHaveBeenCalled(); + expect(h.state.flushPendingInjections).not.toHaveBeenCalled(); + h.ready(); + await settleMicrotasks(); + expect(h.state.inspectCodexUpgradeRuntime).toHaveBeenCalledTimes(1); + expect(h.state.replacementSpawnInProgress).toBe(true); + await h.realFlush(); + expect(h.state.hasPendingInputForFlush).not.toHaveBeenCalled(); + expect(h.state.findCodexRolloutSetByPid).not.toHaveBeenCalled(); + expect(h.state.rawInputRestartGate).toBe(true); + inspected.resolve(replacement); + expect(await done).toBeUndefined(); + await settleMicrotasks(); + expect(h.state.findCodexRolloutSetByPid).toHaveBeenCalledWith(201); + expect(h.state.cliRestartInProgress).toBe(false); + expect(h.state.rawInputRestartGate).toBe(false); + expect(h.state.codexAutoUpgrade).toBeUndefined(); + expect(h.state.replacementSpawnInProgress).toBe(false); + expect(h.state.flushPendingInjections).toHaveBeenCalledTimes(1); + expect(h.state.flushPending).toHaveBeenCalledTimes(1); + expect(h.state.hasPendingInputForFlush).toHaveBeenCalledTimes(1); + expect(h.steps.indexOf('verify thread fd')).toBeLessThan(h.steps.indexOf('flush requested')); + expect(h.steps.indexOf('verify thread fd')).toBeLessThan(h.steps.indexOf('injections flush requested')); + expect(h.steps.indexOf('injections flush requested')).toBeLessThan(h.steps.indexOf('flush requested')); + expect(h.delivered).toEqual([pending]); + }); + + it('keeps accepted inputs and failure gates when old CLI exit cannot be confirmed, with no fresh retry', async () => { + const h = harness(); + const exited = deferred(); + h.state.waitForCodexExit.mockImplementationOnce(() => exited.promise); + const done = h.run(); + await settleMicrotasks(); + const pending = { text: 'accepted while stopping' }; + h.state.pendingMessages.push(pending); + exited.reject(new Error('old Codex is still alive')); + expect(await done).toEqual(new Error('old Codex is still alive')); + expect(h.state.codexAutoUpgrade).toEqual({ stage: 'failed', threadId }); + expect(h.state.cliRestartInProgress).toBe(true); + expect(h.state.rawInputRestartGate).toBe(true); + expect(h.state.replacementSpawnInProgress).toBe(false); + await h.realFlush(); + expect(h.state.pendingMessages).toEqual([pending]); + expect(h.state.spawnCli).not.toHaveBeenCalled(); + expect(h.state.send).toHaveBeenCalledWith(expect.objectContaining({ type: 'user_notify' })); + expect(await h.run()).toBeInstanceOf(Error); + expect(h.oldBackend.destroySession).toHaveBeenCalledTimes(1); + expect(h.state.spawnCli).not.toHaveBeenCalled(); + }); + + it('does not retry a failed spawn as a fresh thread or flush accepted input', async () => { + const h = harness(); + const spawned = deferred(); + h.state.spawnCli.mockImplementationOnce(() => spawned.promise); + const done = h.run(); + await settleMicrotasks(); + h.state.pendingMessages.push({ text: 'keep this turn' }); + spawned.reject(new Error('resume failed')); + expect(await done).toEqual(new Error('resume failed')); + expect(h.state.codexAutoUpgrade).toEqual({ stage: 'failed', threadId }); + expect(h.state.replacementSpawnInProgress).toBe(false); + expect(h.state.pendingMessages).toEqual([{ text: 'keep this turn' }]); + expect(h.state.flushPending).not.toHaveBeenCalled(); + expect(await h.run()).toBeInstanceOf(Error); + expect(h.state.spawnCli).toHaveBeenCalledTimes(1); + expect(vi.getTimerCount()).toBe(0); + }); + + it('fails a replacement that never becomes ready without starting another CLI', async () => { + const h = harness(); + const done = h.run(); + await settleMicrotasks(); + h.state.pendingMessages.push({ text: 'queued until ready' }); + await vi.advanceTimersByTimeAsync(90_000); + expect(await done).toEqual(new Error('New Codex did not become ready')); + expect(h.state.pendingMessages).toEqual([{ text: 'queued until ready' }]); + expect(h.state.codexAutoUpgrade.stage).toBe('failed'); + expect(h.state.cliRestartInProgress).toBe(true); + expect(h.state.replacementSpawnInProgress).toBe(false); + expect(h.state.spawnCli).toHaveBeenCalledTimes(1); + expect(h.state.inspectCodexUpgradeRuntime).not.toHaveBeenCalled(); + expect(h.state.flushPending).not.toHaveBeenCalled(); + }); + + it.each([ + { name: 'wrong version', runtime: [{ ...replacement[0]!, version: '0.146.0' }], fds: [threadId] }, + { name: 'wrong thread fd', runtime: replacement, fds: ['another-thread'] }, + { name: 'shared thread fd owner', runtime: replacement, fds: [threadId, 'another-thread'] }, + { name: 'missing replacement PID', runtime: [], fds: [threadId] }, + ])('keeps inputs fenced after readiness when validation finds $name', async ({ runtime, fds }) => { + const h = harness(); + h.state.inspectCodexUpgradeRuntime.mockResolvedValue(runtime); + h.state.findCodexRolloutSetByPid.mockReturnValue(new Set(fds)); + const done = h.run(); + await settleMicrotasks(); + h.state.pendingMessages.push({ text: 'must stay queued' }); + h.ready(); + expect(await done).toEqual(new Error('Replacement Codex runtime or thread identity did not match')); + expect(h.state.send).not.toHaveBeenCalledWith(expect.objectContaining({ type: 'cli_runtime_version' })); + await h.realFlush(); + expect(h.state.pendingMessages).toEqual([{ text: 'must stay queued' }]); + expect(h.state.codexAutoUpgrade.stage).toBe('failed'); + expect(h.state.cliRestartInProgress).toBe(true); + expect(h.state.rawInputRestartGate).toBe(true); + expect(h.state.replacementSpawnInProgress).toBe(false); + expect(h.state.flushPending).not.toHaveBeenCalled(); + expect(h.state.flushPendingInjections).not.toHaveBeenCalled(); + }); + + it.each(['workingDir', 'env'] as const)('rejects a concurrent %s change before releasing accepted input', async field => { + const h = harness(); + const done = h.run(); + await settleMicrotasks(); + h.state.pendingMessages.push({ text: 'keep this accepted turn' }); + h.state.pendingInjections.push({ kind: 'cwd', cwd: '/new-project' }); + if (field === 'workingDir') h.state.lastInitConfig.workingDir = '/new-project'; + else h.state.lastInitConfig.env = { CODEX_TEST_SETTING: 'changed' }; + h.ready(); + expect(await done).toEqual(new Error('Session launch settings changed during upgrade; restart is required')); + expect(h.state.codexAutoUpgrade).toEqual({ stage: 'failed', threadId }); + expect(h.state.cliRestartInProgress).toBe(true); + expect(h.state.rawInputRestartGate).toBe(true); + expect(h.state.replacementSpawnInProgress).toBe(false); + expect(h.state.pendingMessages).toEqual([{ text: 'keep this accepted turn' }]); + expect(h.state.pendingInjections).toEqual([{ kind: 'cwd', cwd: '/new-project' }]); + expect(h.state.flushPending).not.toHaveBeenCalled(); + expect(h.state.flushPendingInjections).not.toHaveBeenCalled(); + }); + + it('restores RPC ownership and plugin generation before spawning the original thread viewer', async () => { + const h = harness(); + h.state.codexRpcEligible.mockReturnValue(true); + const done = h.run(); + await settleMicrotasks(); + expect(h.state.engageCodexRpc).toHaveBeenCalledWith(expect.objectContaining({ + cliSessionId: threadId, resume: true, prompt: '', forkSession: false, + })); + expect(h.steps.indexOf('plugins')).toBeLessThan(h.steps.indexOf('rpc resume')); + expect(h.steps.indexOf('rpc resume')).toBeLessThan(h.steps.indexOf('spawn')); + expect(h.state.spawnCli).toHaveBeenCalledWith(expect.anything(), { pluginGenerationPrepared: true }); + h.ready(); + expect(await done).toBeUndefined(); + }); + + it.each([ + { outcome: 'not-engaged', remote: threadId }, + { outcome: 'started', remote: threadId }, + { outcome: 'resumed', remote: 'wrong-thread' }, + ])('rejects RPC restoration outcome $outcome / $remote before spawning', async ({ outcome, remote }) => { + const h = harness(); + h.state.codexRpcEligible.mockReturnValue(true); + h.state.engageCodexRpc.mockImplementation(async () => { + h.state.remoteThreadId = remote; + h.state.pendingMessages.push({ text: 'accepted while RPC reconnects' }); + return outcome; + }); + expect(await h.run()).toEqual(new Error('Codex RPC did not resume the original thread')); + expect(h.state.spawnCli).not.toHaveBeenCalled(); + expect(h.state.flushPending).not.toHaveBeenCalled(); + expect(h.state.pendingMessages).toEqual([{ text: 'accepted while RPC reconnects' }]); + expect(h.state.cliRestartInProgress).toBe(true); + expect(h.state.rawInputRestartGate).toBe(true); + expect(h.state.replacementSpawnInProgress).toBe(false); + expect(h.state.flushPendingInjections).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/test/worker-pipe-initial-screen-order.test.ts b/test/worker-pipe-initial-screen-order.test.ts index 294a24fb7..577383ace 100644 --- a/test/worker-pipe-initial-screen-order.test.ts +++ b/test/worker-pipe-initial-screen-order.test.ts @@ -430,7 +430,7 @@ describe('worker pipe initial screen ordering', () => { expect(spawnStart).toBeGreaterThan(-1); expect(prepareIdx).toBeGreaterThan(spawnStart); expect(backendSpawnIdx).toBeGreaterThan(prepareIdx); - expect(source.match(/await spawnCli\(/g)).toHaveLength(3); + expect(source.match(/await spawnCli\(/g)).toHaveLength(4); expect(source.slice(spawnStart, prepareIdx)).toContain('const spawnGeneration = ++cliSpawnGeneration;'); expect(source.slice(prepareIdx, backendSpawnIdx)) .toContain('if (spawnGeneration !== cliSpawnGeneration) throw new CliSpawnSupersededError();'); diff --git a/test/worker-startup-retry-wiring.test.ts b/test/worker-startup-retry-wiring.test.ts index 1e1333b4f..4f246cb66 100644 --- a/test/worker-startup-retry-wiring.test.ts +++ b/test/worker-startup-retry-wiring.test.ts @@ -119,6 +119,7 @@ vi.mock('@larksuiteoapi/node-sdk', () => ({ import { initWorkerPool, __testOnly_setupWorkerHandlers, restartCounts } from '../src/core/worker-pool.js'; import { MAX_STARTUP_AUTO_RETRIES } from '../src/core/worker-startup-retry.js'; +import { dashboardEventBus } from '../src/core/dashboard-events.js'; import type { DaemonSession } from '../src/core/types.js'; function makeFakeWorker() { @@ -177,6 +178,55 @@ async function failOnce(ds: DaemonSession, message: string, extras: Record { + beforeEach(() => { + vi.clearAllMocks(); + initWorkerPool({ + sessionReply: sessionReplyMock, + getSessionWorkingDir: () => '/tmp', + getActiveCount: () => 1, + closeSession: vi.fn(), + } as any); + }); + + it('updates the live session and publishes its version without persisting it', async () => { + const worker = makeFakeWorker(); + const ds = makeDs('sid-version', worker); + ds.cliVersion = '0.146.0'; + __testOnly_setupWorkerHandlers(ds, worker); + vi.mocked(dashboardEventBus.publish).mockClear(); + updateSessionMock.mockClear(); + + worker.emit('message', { type: 'cli_runtime_version', version: '0.153.4' }); + await flush(); + + expect(ds.cliVersion).toBe('0.153.4'); + expect(dashboardEventBus.publish).toHaveBeenCalledExactlyOnceWith({ + type: 'session.update', + body: { sessionId: 'sid-version', patch: { cliVersion: '0.153.4' } }, + }); + expect(updateSessionMock).not.toHaveBeenCalled(); + }); + + it.each(['replacement-worker', 'fenced-generation'] as const)('ignores a runtime version report after ownership changes: %s', async change => { + const worker = makeFakeWorker(); + const ds = makeDs('sid-stale-version', worker); + ds.cliVersion = '0.153.4'; + __testOnly_setupWorkerHandlers(ds, worker); + if (change === 'replacement-worker') ds.worker = makeFakeWorker(); + else ds.session.workerGeneration = (ds.session.workerGeneration ?? 0) + 1; + vi.mocked(dashboardEventBus.publish).mockClear(); + updateSessionMock.mockClear(); + + worker.emit('message', { type: 'cli_runtime_version', version: '0.146.0' }); + await flush(); + + expect(ds.cliVersion).toBe('0.153.4'); + expect(dashboardEventBus.publish).not.toHaveBeenCalled(); + expect(updateSessionMock).not.toHaveBeenCalled(); + }); +}); + describe("worker-pool 'error' transient self-heal wiring", () => { beforeEach(() => { vi.clearAllMocks();