diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 899914339c76..f1da90b6e01f 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -10,6 +10,78 @@ } }, "gates": [ + { + "id": "terminal-session.settled-worker-main-owned-attach", + "title": "Main owns settled worker attachment and preserves fenced pane history", + "maturity": "experimental", + "protection": "partial", + "owner": "terminal-runtime", + "layer": "provider-contract-and-electron-restart", + "surfaces": ["stable pane spawn", "settled worker restart", "paired terminal create"], + "platforms": ["macos", "linux", "windows"], + "providers": ["local", "daemon", "ssh", "remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["local", "daemon", "ssh", "remote-runtime"], + "coverageNotes": "macOS hidden Electron restart exercises live and absent daemon sessions. Unit contracts cover direct SSH evidence and paired host/client refusal; other operating systems and real remote topologies remain unverified.", + "motivatingLinks": ["https://github.com/stablyai/orca/pull/19358"], + "invariant": "Main reads the persisted pane resume fence and permits attachment only. Observed exit preserves the historical tab and binding; unverifiable evidence preserves ownership and permits recovery without replacement.", + "oracle": "A fenced stable owner gets one attach attempt and no retirement or second provider call. A restarted worker with a live or exited daemon session retains its exact tab and PTY binding without another agent launch. A fence committed during attachment or claim reconciliation prevents retirement and fresh provider spawn. Unsupported paired resolution never certifies exit; owner absence cannot create until host fenced-create semantics are negotiated.", + "commands": [ + "ORCA_BACKGROUND_LAUNCH=1 npx vitest run --config config/vitest.config.ts src/main/ipc/pty-attach-only-fenced-pane-spawn.test.ts", + "ORCA_BACKGROUND_LAUNCH=1 npx vitest run --config config/vitest.config.ts src/main/ipc/pty-attach-only-fenced-pane-spawn.test.ts src/main/ipc/pty/pane/stable-pane-resume-fence-host.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-fenced-outcome.test.ts", + "ORCA_BACKGROUND_LAUNCH=1 SKIP_BUILD=1 npx playwright test tests/e2e/settled-worker-tab-survives-restart.spec.ts --config tests/playwright.config.ts --project=electron-headless --workers=1 --retries=0" + ], + "testFiles": [ + "src/main/ipc/pty-attach-only-fenced-pane-spawn.test.ts", + "src/main/ipc/pty/pane/stable-pane-resume-fence-host.test.ts", + "src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-fenced-outcome.test.ts", + "tests/e2e/settled-worker-tab-survives-restart.spec.ts" + ], + "assertionRefs": [ + { + "file": "tests/e2e/settled-worker-tab-survives-restart.spec.ts", + "assertions": [ + "reveal must not tear the settled worker tab down", + "main must report observed exit without retiring the pane" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-09-07", + "runner": "local", + "platform": "macos", + "command": "ORCA_BACKGROUND_LAUNCH=1 npx vitest run --config config/vitest.config.ts src/main/ipc/pty-attach-only-fenced-pane-spawn.test.ts", + "result": "passed", + "durationSeconds": 20.18, + "summary": "Main boundary and adjacent contracts passed 19 tests across three suites." + } + ], + "runtimeBudget": { + "p95Seconds": 120, + "scope": "Two hidden Electron restart cases; unit contracts use injected provider failures." + }, + "flakeHistory": { + "status": "not-started", + "evidence": "Round-2 local validation; CI soak has not begun." + }, + "redGreenEvidence": { + "status": "partial", + "evidence": "Round-1 reviewers reproduced destructive retirement and false exit classification. Round-2 mutation and restart results are recorded in PR #19358." + }, + "performanceBudget": { + "required": true, + "evidence": "Fence lookup reads one host-scoped workspace session. Fenced absence stops after one provider attach with zero persistence retirement or replacement calls; recovery reuses the existing bounded scheduler." + }, + "knownGaps": [ + "No real Windows, Linux, WSL, SSH server, or paired two-process UI journey in this gate.", + "No CI soak history; archived probes requiring renderer attachOnly or contradictory identical-message outcomes cannot serve as the new contract." + ], + "promotionCriteria": [ + "Complete CI soak and add real paired/SSH topology evidence before promotion." + ], + "demotionRule": "Keep experimental while any restart ordering failure is unexplained; never weaken binding retention to silence failures." + }, { "id": "terminal-performance.padded-fullscreen-redraw", "title": "Fullscreen redraw padding does not stall terminal delivery", diff --git a/src/main/daemon/daemon-errors.ts b/src/main/daemon/daemon-errors.ts index 12136d37a5bd..ff3ce0ede97b 100644 --- a/src/main/daemon/daemon-errors.ts +++ b/src/main/daemon/daemon-errors.ts @@ -1,5 +1,7 @@ // Error classes shared across the daemon protocol boundary (client, server, // host). Split from types.ts, which is capped for wire-shape declarations. +import { SESSION_NOT_FOUND_MESSAGE_PREFIX } from '../../shared/pty-attach-absence-evidence' + const ATTACH_CANCELED_PREFIX = 'Attach canceled for session ' export class TerminalAttachCanceledError extends Error { @@ -56,7 +58,7 @@ export const DAEMON_UNAVAILABLE_RECONNECT_MESSAGE = 'Daemon temporarily unavaila export class SessionNotFoundError extends Error { constructor(sessionId: string) { - super(`Session not found: ${sessionId}`) + super(`${SESSION_NOT_FOUND_MESSAGE_PREFIX}${sessionId}`) this.name = 'SessionNotFoundError' } } @@ -87,8 +89,7 @@ export function isDaemonEndpointGoneError(err: unknown): boolean { } export function decodeDaemonResponseError(message: string): Error { - const prefix = 'Session not found: ' - return message.startsWith(prefix) - ? new SessionNotFoundError(message.slice(prefix.length)) + return message.startsWith(SESSION_NOT_FOUND_MESSAGE_PREFIX) + ? new SessionNotFoundError(message.slice(SESSION_NOT_FOUND_MESSAGE_PREFIX.length)) : new DaemonProtocolError(message) } diff --git a/src/main/daemon/daemon-pty-adapter-attach-only-ownership.test.ts b/src/main/daemon/daemon-pty-adapter-attach-only-ownership.test.ts new file mode 100644 index 000000000000..fae84b17f38c --- /dev/null +++ b/src/main/daemon/daemon-pty-adapter-attach-only-ownership.test.ts @@ -0,0 +1,63 @@ +/* A fresh adapter (new app process) taking over a session an older process spawned. */ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { rmSync } from 'node:fs' +import { DaemonPtyAdapter } from './daemon-pty-adapter' +import { SessionNotFoundError } from './daemon-errors' +import { createMockSubprocess, startDaemonAdapterHarness } from './daemon-pty-adapter-test-harness' + +// After an app restart a settled orchestration worker's pane reattaches under `attachOnly`. That +// reattach is what makes the restarted main process an owner of the daemon session, so every +// downstream consumer of the adapter cache — `hasPty`, resize, write, liveness, the serializer — +// sees a session main actually owns. The pane used to attach renderer-only, leaving this cache +// empty and `pty:hasPty` fabricating an absence the renderer closed the tab on (#16904 regression). +describe('DaemonPtyAdapter attach-only ownership after a restart', () => { + let harness: Awaited> + let restartedAdapter: DaemonPtyAdapter + + beforeEach(async () => { + harness = await startDaemonAdapterHarness(() => createMockSubprocess()) + restartedAdapter = new DaemonPtyAdapter({ + socketPath: harness.socketPath, + tokenPath: harness.tokenPath + }) + }) + afterEach(async () => { + restartedAdapter.dispose() + harness.adapter.dispose() + await harness.server.shutdown() + rmSync(harness.dir, { recursive: true, force: true }) + }) + + it('owns the session after an attach-only spawn, so the cache alone answers present', async () => { + const { id } = await harness.adapter.spawn({ cols: 80, rows: 24 }) + expect(restartedAdapter.hasPty(id)).toBe(false) + + const attached = await restartedAdapter.spawn({ + cols: 80, + rows: 24, + sessionId: id, + attachOnly: true + }) + + expect(attached.id).toBe(id) + expect(attached.isReattach).toBe(true) + expect(restartedAdapter.hasPty(id)).toBe(true) + }) + + it('refuses to create a session the daemon does not have', async () => { + await expect( + restartedAdapter.spawn({ cols: 80, rows: 24, sessionId: 'never-spawned', attachOnly: true }) + ).rejects.toBeInstanceOf(SessionNotFoundError) + expect(restartedAdapter.hasPty('never-spawned')).toBe(false) + }) + + it('refuses to replace a session that exited before the attach', async () => { + const { id } = await harness.adapter.spawn({ cols: 80, rows: 24 }) + await harness.adapter.shutdown(id, { immediate: true }) + + await expect( + restartedAdapter.spawn({ cols: 80, rows: 24, sessionId: id, attachOnly: true }) + ).rejects.toBeInstanceOf(SessionNotFoundError) + expect(restartedAdapter.hasPty(id)).toBe(false) + }) +}) diff --git a/src/main/ipc/pty-attach-only-fenced-pane-spawn.test.ts b/src/main/ipc/pty-attach-only-fenced-pane-spawn.test.ts new file mode 100644 index 000000000000..bce6f88aa406 --- /dev/null +++ b/src/main/ipc/pty-attach-only-fenced-pane-spawn.test.ts @@ -0,0 +1,482 @@ +import { describe, expect, it, vi } from 'vitest' +import { setupPtyIpcSuite } from './pty-ipc-test-harness' +import { SessionNotFoundError } from '../daemon/daemon-errors' +import { makePaneKey } from '../../shared/stable-pane-id' +import { registerPtyHandlers } from './pty' +import * as stableAdoption from './pty/pane/adopt-stable' +import { ptySizes } from './pty/delivery/visibility-state' +import { isHiddenRendererPty } from './pty-hidden-delivery-gate' +import { paneSpawnReservationsByOwnerKey } from './pty/pane/spawn-reservation' + +vi.mock('electron', () => import('./pty-ipc-mock-registry').then((m) => m.electronModuleMock())) +vi.mock('fs', () => import('./pty-ipc-mock-registry').then((m) => m.fsModuleMock())) +vi.mock('node-pty', () => import('./pty-ipc-mock-registry').then((m) => m.nodePtyModuleMock())) +vi.mock('node:child_process', async (importOriginal) => + (await import('./pty-ipc-mock-registry')).childProcessModuleMock(await importOriginal()) +) +vi.mock('../opencode/hook-service', () => + import('./pty-ipc-mock-registry').then((m) => m.openCodeHookServiceModuleMock()) +) +vi.mock('../mimo/hook-service', () => + import('./pty-ipc-mock-registry').then((m) => m.mimoHookServiceModuleMock()) +) +vi.mock('../agent-hooks/server', () => + import('./pty-ipc-mock-registry').then((m) => m.agentHookServerModuleMock()) +) +vi.mock('../pi/titlebar-extension-service', () => + import('./pty-ipc-mock-registry').then((m) => m.piTitlebarExtensionModuleMock()) +) +vi.mock('../pwsh', () => import('./pty-ipc-mock-registry').then((m) => m.pwshModuleMock())) +vi.mock('../wsl', async (importOriginal) => + (await import('./pty-ipc-mock-registry')).wslModuleMock(await importOriginal()) +) +vi.mock('../telemetry/client', () => + import('./pty-ipc-mock-registry').then((m) => m.telemetryClientModuleMock()) +) +vi.mock('../telemetry/classify-error', () => + import('./pty-ipc-mock-registry').then((m) => m.classifyErrorModuleMock()) +) +vi.mock('../cli/linux-terminal-orca-cli-shim', () => + import('./pty-ipc-mock-registry').then((m) => m.linuxCliShimModuleMock()) +) +vi.mock('../memory/pty-registry', () => + import('./pty-ipc-mock-registry').then((m) => m.ptyRegistryModuleMock()) +) +vi.mock('../agent-hooks/migration-unsupported-pty-state', () => + import('./pty-ipc-mock-registry').then((m) => m.migrationUnsupportedPtyModuleMock()) +) +vi.mock('../codex/codex-pane-account-registry', () => + import('./pty-ipc-mock-registry').then((m) => m.codexPaneAccountRegistryModuleMock()) +) +vi.mock('../codex/codex-state-db-backfill-recovery', () => + import('./pty-ipc-mock-registry').then((m) => m.codexBackfillRecoveryModuleMock()) +) + +describe('pty:spawn under a persisted main-owned resume fence', () => { + const { + handlers, + mainWindow, + installDaemonTestProvider, + recoveredAgentClaim, + recoveredAgentSurface + } = setupPtyIpcSuite() + + function buildFencedPaneContext(name: string) { + const worktreeId = `repo-1::/tmp/${name}` + const tabId = `tab-${name}` + const leafId = '5b5b5b5b-5b5b-4b5b-8b5b-5b5b5b5b5b5b' + const ptyId = `pty-${name}` + const paneKey = makePaneKey(tabId, leafId) + let session = { + tabsByWorktree: { [worktreeId]: [{ id: tabId, worktreeId, ptyId }] }, + terminalLayoutsByTabId: { + [tabId]: { + root: { type: 'leaf' as const, leafId }, + activeLeafId: leafId, + expandedLeafId: null, + ptyIdsByLeafId: { [leafId]: ptyId } as Record + } + }, + sleepingAgentSessionsByPaneKey: { + [paneKey]: { worktreeId, automaticResumeBlockedBy: 'legacy-orchestration-worker' } + }, + terminalPtyIncarnationsByPaneKey: { [paneKey]: `inc-${name}` } + } + const store = { + getWorkspaceSession: vi.fn(() => session), + setWorkspaceSession: vi.fn((next: typeof session) => { + session = next + }), + flushOrThrow: vi.fn(), + persistPtyBinding: vi.fn(), + getFolderWorkspace: vi.fn(() => undefined), + getFolderWorkspaces: vi.fn(() => []), + getProjectGroups: vi.fn(() => []), + getRepos: vi.fn(() => []) + } + const runtime = { + setPtyController: vi.fn((_controller: unknown) => {}), + resolveTerminalPane: vi.fn(() => { + throw new Error('terminal_not_found') + }), + createPreAllocatedTerminalHandle: vi.fn(() => `term-${name}`), + preAllocateHandleForPty: vi.fn(() => `term-${name}`), + registerPreAllocatedHandleForPty: vi.fn(), + beginPtyRegistration: vi.fn(), + cancelPendingPtyRegistration: vi.fn(), + assertPtyRegistrationAllowed: vi.fn(), + preparePtyExecutionContext: vi.fn(), + registerPty: vi.fn(), + noteTerminalSpawnCommand: vi.fn(), + seedHeadlessTerminal: vi.fn(), + onPtySpawned: vi.fn(), + onPtyExit: vi.fn(), + onPtyData: vi.fn() + } + const spawnArgs = { + cols: 80, + rows: 24, + cwd: `/tmp/${name}`, + worktreeId, + tabId, + leafId, + sessionId: ptyId, + initiallyHidden: true, + env: { ORCA_PANE_KEY: paneKey, ORCA_TAB_ID: tabId, ORCA_WORKTREE_ID: worktreeId } + } + return { ptyId, store, runtime, spawnArgs } + } + + it('attaches the restored session so main owns it, without a fresh spawn', async () => { + const { ptyId, store, runtime, spawnArgs } = buildFencedPaneContext('fenced-live-worker') + const providerSpawn = vi.fn(async (options: { attachOnly?: boolean; sessionId?: string }) => { + if (!options.attachOnly) { + throw new Error('a fenced pane must never reach a fresh spawn') + } + return { id: options.sessionId!, incarnationId: 'inc-fenced-live-worker', isReattach: true } + }) + installDaemonTestProvider({ spawn: providerSpawn }) + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + + await expect(handlers.get('pty:spawn')!(null, spawnArgs)).resolves.toMatchObject({ id: ptyId }) + expect(providerSpawn.mock.calls.every(([options]) => options.attachOnly === true)).toBe(true) + expect(runtime.onPtyExit).not.toHaveBeenCalled() + }) + + it('refuses to mint a replacement session when the owner proves the session absent', async () => { + const { store, runtime, spawnArgs } = buildFencedPaneContext('fenced-absent-worker') + const providerSpawn = vi.fn(async (options: { attachOnly?: boolean; sessionId?: string }) => { + if (options.attachOnly) { + throw new SessionNotFoundError(options.sessionId ?? '') + } + return { id: 'pty-replacement', incarnationId: 'inc-replacement' } + }) + installDaemonTestProvider({ spawn: providerSpawn }) + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + + await expect(handlers.get('pty:spawn')!(null, spawnArgs)).resolves.toEqual({ + id: spawnArgs.sessionId, + exitedBeforeAttach: true + }) + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + expect(store.persistPtyBinding).not.toHaveBeenCalled() + expect(runtime.beginPtyRegistration).not.toHaveBeenCalled() + expect(isHiddenRendererPty(spawnArgs.sessionId)).toBe(false) + expect(runtime.registerPty).not.toHaveBeenCalled() + expect(runtime.assertPtyRegistrationAllowed).not.toHaveBeenCalled() + expect(runtime.preparePtyExecutionContext).not.toHaveBeenCalled() + expect(runtime.onPtyExit).not.toHaveBeenCalled() + expect(providerSpawn).toHaveBeenCalledTimes(1) + expect(providerSpawn.mock.calls.every(([options]) => options.attachOnly === true)).toBe(true) + }) + + it('allows an unfenced fresh tab to spawn', async () => { + const { store, runtime, spawnArgs } = buildFencedPaneContext('fenced-no-session') + const providerSpawn = installDaemonTestProvider() + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + + await handlers.get('pty:spawn')!(null, { + ...spawnArgs, + sessionId: undefined, + tabId: 'tab-unbound', + leafId: '6c6c6c6c-6c6c-4c6c-8c6c-6c6c6c6c6c6c' + }) + + expect(providerSpawn.mock.calls.at(-1)?.[0]).not.toMatchObject({ attachOnly: true }) + }) + it.each([new Error('daemon timeout'), new Error('Session not found: untyped')])( + 'preserves every binding on unverifiable local evidence: %s', + async (error) => { + const { store, runtime, spawnArgs } = buildFencedPaneContext('unverifiable-worker') + const providerSpawn = vi.fn(async () => { + throw error + }) + installDaemonTestProvider({ spawn: providerSpawn }) + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + await expect(handlers.get('pty:spawn')!(null, spawnArgs)).resolves.toEqual({ + id: spawnArgs.sessionId, + reattachUnverifiable: true + }) + expect(providerSpawn).toHaveBeenCalledTimes(1) + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + expect(store.persistPtyBinding).not.toHaveBeenCalled() + expect(runtime.registerPty).not.toHaveBeenCalled() + expect(runtime.assertPtyRegistrationAllowed).not.toHaveBeenCalled() + expect(runtime.preparePtyExecutionContext).not.toHaveBeenCalled() + expect(runtime.onPtyExit).not.toHaveBeenCalled() + } + ) + + it.each([true, false])( + 'never spawns an ownerless fenced pane (session supplied: %s)', + async (supplied) => { + const { store, runtime, spawnArgs } = buildFencedPaneContext('ownerless-worker') + store.getWorkspaceSession().terminalLayoutsByTabId[spawnArgs.tabId].ptyIdsByLeafId = {} + const providerSpawn = installDaemonTestProvider() + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + await expect( + handlers.get('pty:spawn')!(null, { + ...spawnArgs, + sessionId: supplied ? spawnArgs.sessionId : undefined + }) + ).resolves.toMatchObject({ reattachUnverifiable: true }) + expect(providerSpawn).not.toHaveBeenCalled() + expect(runtime.beginPtyRegistration).not.toHaveBeenCalled() + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + } + ) + + it.each([true, false])( + 'runtime controller refuses ownerless fenced pane (session supplied: %s)', + async (supplied) => { + const { store, runtime, spawnArgs } = buildFencedPaneContext('runtime-ownerless-worker') + store.getWorkspaceSession().terminalLayoutsByTabId[spawnArgs.tabId].ptyIdsByLeafId = {} + const providerSpawn = installDaemonTestProvider() + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + const controller = runtime.setPtyController.mock.calls[0]![0] as { + spawn: (args: unknown) => Promise + } + await expect( + controller.spawn({ ...spawnArgs, sessionId: supplied ? spawnArgs.sessionId : undefined }) + ).resolves.toMatchObject({ reattachUnverifiable: true }) + expect(providerSpawn).not.toHaveBeenCalled() + expect(runtime.beginPtyRegistration).not.toHaveBeenCalled() + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + } + ) + it('settles execute-stage refusal without registration, hidden marks or size changes', async () => { + const { store, runtime, spawnArgs } = buildFencedPaneContext('execute-refusal') + const earlyAdoption = vi.spyOn(stableAdoption, 'adoptStablePane').mockResolvedValueOnce(null) + const providerSpawn = vi.fn(async () => { + runtime.preparePtyExecutionContext.mockClear() + throw new SessionNotFoundError(spawnArgs.sessionId) + }) + installDaemonTestProvider({ spawn: providerSpawn }) + ptySizes.set(spawnArgs.sessionId, { cols: 120, rows: 40 }) + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + try { + await expect( + handlers.get('pty:spawn')!(null, { ...spawnArgs, initiallyHidden: true }) + ).resolves.toEqual({ id: spawnArgs.sessionId, exitedBeforeAttach: true }) + expect(providerSpawn).toHaveBeenCalledTimes(1) + expect(runtime.cancelPendingPtyRegistration).toHaveBeenCalledWith(spawnArgs.sessionId) + expect(runtime.registerPty).not.toHaveBeenCalled() + expect(runtime.assertPtyRegistrationAllowed).not.toHaveBeenCalled() + expect(runtime.preparePtyExecutionContext).not.toHaveBeenCalled() + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + expect(store.persistPtyBinding).not.toHaveBeenCalled() + expect(isHiddenRendererPty(spawnArgs.sessionId)).toBe(false) + expect(ptySizes.get(spawnArgs.sessionId)).toEqual({ cols: 120, rows: 40 }) + expect(paneSpawnReservationsByOwnerKey.size).toBe(0) + } finally { + earlyAdoption.mockRestore() + } + }) + it.each([false, true])( + 'runtime controller carries a fenced owner refusal (claim: %s)', + async (ensureClaim) => { + const { store, runtime, spawnArgs } = buildFencedPaneContext('runtime-absent-worker') + const providerSpawn = vi.fn(async () => { + runtime.preparePtyExecutionContext.mockClear() + throw new SessionNotFoundError(spawnArgs.sessionId) + }) + installDaemonTestProvider({ spawn: providerSpawn }) + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + const controller = runtime.setPtyController.mock.calls[0]![0] as { + spawn: (args: unknown) => Promise + } + await expect( + controller.spawn({ + ...spawnArgs, + ...(ensureClaim + ? { agentSessionEnsure: { claim: recoveredAgentClaim, surface: recoveredAgentSurface } } + : {}) + }) + ).resolves.toEqual({ + id: spawnArgs.sessionId, + exitedBeforeAttach: true + }) + expect(providerSpawn).toHaveBeenCalledTimes(1) + expect(runtime.registerPty).not.toHaveBeenCalled() + expect(runtime.assertPtyRegistrationAllowed).not.toHaveBeenCalled() + expect(runtime.preparePtyExecutionContext).not.toHaveBeenCalled() + expect(runtime.cancelPendingPtyRegistration).toHaveBeenCalledWith(spawnArgs.sessionId) + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + expect(store.persistPtyBinding).not.toHaveBeenCalled() + expect(paneSpawnReservationsByOwnerKey.size).toBe(0) + } + ) + it.each([false, true])( + 'carries unverifiable through the runtime controller (pre-adopted: %s)', + async (preAdopted) => { + const { store, runtime, spawnArgs } = buildFencedPaneContext('unverifiable-controller') + const spawn = vi.fn(async () => { + throw new Error('daemon unavailable') + }) + installDaemonTestProvider({ spawn }) + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + const controller = runtime.setPtyController.mock.calls[0]![0] as { + spawn: (args: unknown) => Promise + } + const result = await controller.spawn({ + ...spawnArgs, + ...(preAdopted + ? { + adoptedStablePane: { + result: { id: spawnArgs.sessionId, reattachUnverifiable: true }, + owner: { + ptyId: spawnArgs.sessionId, + tabId: spawnArgs.tabId, + leafId: spawnArgs.leafId + } + } + } + : {}) + }) + expect(result).toEqual({ id: spawnArgs.sessionId, reattachUnverifiable: true }) + expect(runtime.assertPtyRegistrationAllowed).not.toHaveBeenCalled() + if (preAdopted) { + expect(spawn).not.toHaveBeenCalled() + expect(runtime.beginPtyRegistration).not.toHaveBeenCalled() + expect(runtime.preparePtyExecutionContext).not.toHaveBeenCalled() + } + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + } + ) + it.each(['ipc', 'runtime'] as const)( + 'preserves a fence committed during %s attach', + async (entry) => { + const { store, runtime, spawnArgs } = buildFencedPaneContext('late-fenced-worker') + const record = + store.getWorkspaceSession().sleepingAgentSessionsByPaneKey[ + makePaneKey(spawnArgs.tabId, spawnArgs.leafId) + ] + record.automaticResumeBlockedBy = '' + const spawn = vi.fn(async () => { + record.automaticResumeBlockedBy = 'legacy-orchestration-worker' + throw new SessionNotFoundError(spawnArgs.sessionId) + }) + installDaemonTestProvider({ spawn }) + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + const controller = runtime.setPtyController.mock.calls[0]![0] as { + spawn: (args: unknown) => Promise + } + const result = + entry === 'ipc' + ? await handlers.get('pty:spawn')!(null, spawnArgs) + : await controller.spawn(spawnArgs) + expect(result).toEqual({ id: spawnArgs.sessionId, reattachUnverifiable: true }) + expect(spawn).toHaveBeenCalledTimes(1) + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + expect(runtime.onPtyExit).not.toHaveBeenCalled() + } + ) + it('refuses a claim spawn when the fence commits during owner reconciliation', async () => { + const { store, runtime, spawnArgs } = buildFencedPaneContext('late-claim-fence') + const record = + store.getWorkspaceSession().sleepingAgentSessionsByPaneKey[ + makePaneKey(spawnArgs.tabId, spawnArgs.leafId) + ] + record.automaticResumeBlockedBy = '' + const spawn = vi.fn(async () => ({ id: 'replacement' })) + const listProcesses = vi.fn(async () => { + record.automaticResumeBlockedBy = 'legacy-orchestration-worker' + return [] + }) + installDaemonTestProvider({ spawn, listProcesses }) + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + const controller = runtime.setPtyController.mock.calls[0]![0] as { + spawn: (args: unknown) => Promise + } + await expect( + controller.spawn({ + ...spawnArgs, + agentSessionEnsure: { claim: recoveredAgentClaim, surface: recoveredAgentSurface } + }) + ).resolves.toEqual({ id: spawnArgs.sessionId, reattachUnverifiable: true }) + expect(listProcesses).toHaveBeenCalled() + expect(spawn).not.toHaveBeenCalled() + expect(runtime.registerPty).not.toHaveBeenCalled() + expect(runtime.assertPtyRegistrationAllowed).not.toHaveBeenCalled() + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + expect(runtime.cancelPendingPtyRegistration).toHaveBeenCalledWith(spawnArgs.sessionId) + expect(paneSpawnReservationsByOwnerKey.size).toBe(0) + }) +}) diff --git a/src/main/ipc/pty-runtime-kill-and-exit.test.ts b/src/main/ipc/pty-runtime-kill-and-exit.test.ts index 2200b2b6043d..3786ee3c23f3 100644 --- a/src/main/ipc/pty-runtime-kill-and-exit.test.ts +++ b/src/main/ipc/pty-runtime-kill-and-exit.test.ts @@ -542,4 +542,29 @@ describe('registerPtyHandlers', () => { vi.useRealTimers() } }) + it.each(['renderer', 'runtime'] as const)( + '%s kill treats daemon session-not-found as completed shutdown', + async (entry) => { + const id = 'daemon-missing-session' + const shutdown = vi.fn(async () => { + throw new Error(`Session not found: ${id}`) + }) + installDaemonTestProvider({ shutdown }) + setPtyOwnership(id, null) + const runtime = { setPtyController: vi.fn(), onPtyExit: vi.fn() } + registerPtyHandlers(mainWindow as never, runtime as never) + if (entry === 'renderer') { + await expect(handlers.get('pty:kill')!(null, { id })).resolves.toBeUndefined() + } else { + const controller = runtime.setPtyController.mock.calls[0]![0] as { + kill: (id: string) => boolean + } + expect(controller.kill(id)).toBe(true) + } + await vi.waitFor(() => expect(runtime.onPtyExit).toHaveBeenCalled()) + expect(shutdown).toHaveBeenCalledTimes(1) + const { ptyOwnership } = await import('./pty/provider/ownership-state') + expect(ptyOwnership.has(id)).toBe(false) + } + ) }) diff --git a/src/main/ipc/pty/ipc/spawn-begin.ts b/src/main/ipc/pty/ipc/spawn-begin.ts index 0b59d347e4dc..b72025673126 100644 --- a/src/main/ipc/pty/ipc/spawn-begin.ts +++ b/src/main/ipc/pty/ipc/spawn-begin.ts @@ -12,6 +12,7 @@ import { paneSpawnReservationsByOwnerKey, pendingRuntimePaneCreatesByOwnerKey } from '../pane/spawn-reservation' +import { isStablePaneResumeBlocked } from '../pane/stable-pane-resume-fence' import { resolveStablePaneOwner } from '../pane/stable-owner' import type { PtyIpcSpawnState } from './spawn-state' @@ -96,6 +97,12 @@ export async function beginPtyIpcSpawn( args.connectionId ) : null + if ( + !ctx.earlyStablePaneOwner && + isStablePaneResumeBlocked(ctx.deps.store, earlyPaneKey, args.worktreeId, args.connectionId) + ) { + return { id: args.sessionId ?? '', reattachUnverifiable: true } + } ctx.earlyWorktreeId = args.worktreeId // Reserve early so renderer/runtime materialization cannot start duplicate provider spawns. ctx.paneSpawnReservationKey = earlyReservationKey diff --git a/src/main/ipc/pty/ipc/spawn-execute.ts b/src/main/ipc/pty/ipc/spawn-execute.ts index 2d9de676c39d..c6fad43d1ff1 100644 --- a/src/main/ipc/pty/ipc/spawn-execute.ts +++ b/src/main/ipc/pty/ipc/spawn-execute.ts @@ -67,6 +67,9 @@ export async function executePtyIpcSpawn(ctx: PtyIpcSpawnState): Promise { }) ctx.result = stablePaneSpawn.result ctx.stablePaneOwner = stablePaneSpawn.owner + if (ctx.result.exitedBeforeAttach || ctx.result.reattachUnverifiable) { + return + } if ( ctx.stablePaneOwner && ctx.isMintedSessionId && diff --git a/src/main/ipc/pty/ipc/spawn-run.ts b/src/main/ipc/pty/ipc/spawn-run.ts index 82e2d33f3830..3024d4542132 100644 --- a/src/main/ipc/pty/ipc/spawn-run.ts +++ b/src/main/ipc/pty/ipc/spawn-run.ts @@ -1,4 +1,4 @@ -import { rejectPaneSpawnReservation } from '../pane/spawn-reservation' +import { rejectPaneSpawnReservation, resolvePaneSpawnReservation } from '../pane/spawn-reservation' import { ptySizes } from '../delivery/visibility-state' import { beginPtyIpcSpawn } from './spawn-begin' import { preparePtyIpcSpawnPreflight } from './spawn-preflight' @@ -39,6 +39,17 @@ export async function runPtyIpcSpawn(deps: PtySpawnIpcDeps, args: PtySpawnIpcArg } try { await preparePtyIpcSpawnPreflight(ctx) + const attachOutcome = ctx.preAdoptedStablePane?.result + if (attachOutcome?.exitedBeforeAttach || attachOutcome?.reattachUnverifiable) { + if (ctx.preSpawnHiddenMarkId !== null) { + ctx.deps.transitionSpawnHiddenRendererPtyDeliveryState(ctx.preSpawnHiddenMarkId, false) + } + return resolvePaneSpawnReservation( + ctx.paneSpawnReservationKey, + ctx.paneSpawnReservation, + attachOutcome + ) + } await assemblePtyIpcSpawnEnv(ctx) const earlyReserved = await buildPtyIpcSpawnOptions(ctx).catch((error: unknown) => { restoreProvisionalPtySize(ctx) @@ -53,6 +64,20 @@ export async function runPtyIpcSpawn(deps: PtySpawnIpcDeps, args: PtySpawnIpcArg return earlyReserved } await executePtyIpcSpawn(ctx) + if (ctx.result.exitedBeforeAttach || ctx.result.reattachUnverifiable) { + releaseAbandonedAgentTeamsLeader(ctx) + restoreProvisionalPtySize(ctx) + if (ctx.preSpawnHiddenMarkId !== null) { + ctx.deps.transitionSpawnHiddenRendererPtyDeliveryState(ctx.preSpawnHiddenMarkId, false) + } + deps.runtime?.cancelPendingPtyRegistration?.(ctx.pendingRegistrationPtyId ?? ctx.result.id) + ctx.pendingRegistrationPtyId = null + return resolvePaneSpawnReservation( + ctx.paneSpawnReservationKey, + ctx.paneSpawnReservation, + ctx.result + ) + } return await commitPtyIpcSpawn(ctx) } catch (err) { releaseAbandonedAgentTeamsLeader(ctx) diff --git a/src/main/ipc/pty/pane/stable-owner-retirement.ts b/src/main/ipc/pty/pane/stable-owner-retirement.ts new file mode 100644 index 000000000000..4bf7ccf75f7c --- /dev/null +++ b/src/main/ipc/pty/pane/stable-owner-retirement.ts @@ -0,0 +1,42 @@ +import { toSshExecutionHostId } from '../../../../shared/execution-host' +import { makePaneKey } from '../../../../shared/stable-pane-id' +import type { Store } from '../../../persistence' +import { retireTerminalSurfaceFromPersistence } from '../../../runtime/mobile-session-terminal-persistence-retirement' +import { resolvePersistedStablePaneOwner, type StablePaneOwner } from './stable-owner' + +export function retirePersistedStablePaneOwner( + store: Store | undefined, + owner: StablePaneOwner, + worktreeId: string, + connectionId: string | null | undefined +): boolean { + if (!store) { + return false + } + const paneKey = makePaneKey(owner.tabId, owner.leafId) + const hostId = connectionId ? toSshExecutionHostId(connectionId) : undefined + const current = resolvePersistedStablePaneOwner(store, paneKey, worktreeId, connectionId) + if (!current) { + // Why: persistence already dropped this pane binding (an earlier stop retired it while the + // runtime kept history), so there is nothing left to clear — that is a completed retirement, + // not a competing owner. Reporting failure here strands the pane after its PTY is proven dead. + return true + } + if (current.ptyId !== owner.ptyId || current.incarnationId !== owner.persistedIncarnationId) { + return false + } + const session = store.getWorkspaceSession(hostId) + const retired = retireTerminalSurfaceFromPersistence(session, { + worktreeId, + parentTabId: owner.tabId, + leafId: owner.leafId, + ptyId: owner.ptyId, + ...(current.incarnationId ? { incarnationId: current.incarnationId } : {}) + }) + if (retired === session) { + return false + } + store.setWorkspaceSession(retired, hostId) + store.flushOrThrow() + return true +} diff --git a/src/main/ipc/pty/pane/stable-owner.ts b/src/main/ipc/pty/pane/stable-owner.ts index 731065e59ab3..6f1574833d11 100644 --- a/src/main/ipc/pty/pane/stable-owner.ts +++ b/src/main/ipc/pty/pane/stable-owner.ts @@ -1,8 +1,9 @@ +import { isStablePaneResumeBlocked } from './stable-pane-resume-fence' import { toSshExecutionHostId } from '../../../../shared/execution-host' import { makePaneKey, parsePaneKey } from '../../../../shared/stable-pane-id' import { UNVERIFIED_PROCESS_EXIT_CODE } from '../../../../shared/terminal-exit-cause' import type { Store } from '../../../persistence' -import { retireTerminalSurfaceFromPersistence } from '../../../runtime/mobile-session-terminal-persistence-retirement' +import { retirePersistedStablePaneOwner } from './stable-owner-retirement' import type { OrcaRuntimeService } from '../../../runtime/orca-runtime' import type { IPtyProvider, PtySpawnOptions, PtySpawnResult } from '../../../providers/types' import { parseAppSshPtyId } from '../../../providers/ssh-pty-id' @@ -120,43 +121,6 @@ export function resolveStablePaneOwner( } } -export function retirePersistedStablePaneOwner( - store: Store | undefined, - owner: StablePaneOwner, - worktreeId: string, - connectionId: string | null | undefined -): boolean { - if (!store) { - return false - } - const paneKey = makePaneKey(owner.tabId, owner.leafId) - const hostId = connectionId ? toSshExecutionHostId(connectionId) : undefined - const current = resolvePersistedStablePaneOwner(store, paneKey, worktreeId, connectionId) - if (!current) { - // Why: persistence already dropped this pane binding (an earlier stop retired it while the - // runtime kept history), so there is nothing left to clear — that is a completed retirement, - // not a competing owner. Reporting failure here strands the pane after its PTY is proven dead. - return true - } - if (current.ptyId !== owner.ptyId || current.incarnationId !== owner.persistedIncarnationId) { - return false - } - const session = store.getWorkspaceSession(hostId) - const retired = retireTerminalSurfaceFromPersistence(session, { - worktreeId, - parentTabId: owner.tabId, - leafId: owner.leafId, - ptyId: owner.ptyId, - ...(current.incarnationId ? { incarnationId: current.incarnationId } : {}) - }) - if (retired === session) { - return false - } - store.setWorkspaceSession(retired, hostId) - store.flushOrThrow() - return true -} - export type StablePaneSpawnContext = { runtime: OrcaRuntimeService | undefined store?: Store @@ -214,6 +178,13 @@ export async function attachStablePaneOwner( args: StablePaneSpawnContext & { owner: StablePaneOwner } ): Promise<{ result: PtySpawnResult; owner: StablePaneOwner } | null> { const { owner, provider, runtime, spawnOptions } = args + const paneKey = makePaneKey(owner.tabId, owner.leafId) + const blockedAtAttach = isStablePaneResumeBlocked( + args.store, + paneKey, + args.worktreeId, + args.connectionId + ) let result: PtySpawnResult try { result = await provider.spawn({ @@ -233,6 +204,17 @@ export async function attachStablePaneOwner( onPtySpawnCommitted: undefined }) } catch (error) { + if (isStablePaneResumeBlocked(args.store, paneKey, args.worktreeId, args.connectionId)) { + return { + owner, + result: { + id: owner.ptyId, + ...(blockedAtAttach && isObservedPtyExitEvidence(error) + ? { exitedBeforeAttach: true as const } + : { reattachUnverifiable: true as const }) + } + } + } if (error instanceof TerminalSessionOwnerUnverifiedError) { throw new Error('terminal_pane_owner_unverified') } @@ -298,6 +280,19 @@ export async function spawnForStablePane( return attached } } + if ( + isStablePaneResumeBlocked( + args.store, + args.spawnOptions.paneKey, + args.worktreeId, + args.connectionId + ) + ) { + return { + result: { id: args.spawnOptions.sessionId ?? '', reattachUnverifiable: true }, + owner: null + } + } const result = await args.provider.spawn(args.spawnOptions) args.onFreshSpawn?.(result) return { result, owner: null } diff --git a/src/main/ipc/pty/pane/stable-pane-fenced-evidence.test.ts b/src/main/ipc/pty/pane/stable-pane-fenced-evidence.test.ts new file mode 100644 index 000000000000..22a4ce203ca3 --- /dev/null +++ b/src/main/ipc/pty/pane/stable-pane-fenced-evidence.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it, vi } from 'vitest' +import { getDefaultWorkspaceSession } from '../../../../shared/constants' +import { makePaneKey } from '../../../../shared/stable-pane-id' +import type { Store } from '../../../persistence' +import { SessionNotFoundError } from '../../../daemon/daemon-errors' +import { + SshPtyAbsentFromRelayError, + SshPtyProvenExitedOnRelayError +} from '../../../providers/ssh-pty-errors' +import type { IPtyProvider } from '../../../providers/types' +import { resolveStablePaneOwner, spawnForStablePane } from './stable-owner' +import { ptyOwnership } from '../provider/ownership-state' + +const tabId = 'tab-worker' +const leafId = '5b5b5b5b-5b5b-4b5b-8b5b-5b5b5b5b5b5b' +const paneKey = makePaneKey(tabId, leafId) +const worktreeId = 'folder-worker' + +function fixture(connectionId: string | null) { + const ptyId = connectionId ? `ssh:${connectionId}@@worker` : 'worker' + const session = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [worktreeId]: [{ id: tabId, worktreeId, type: 'terminal' }] }, + terminalLayoutsByTabId: { [tabId]: { ptyIdsByLeafId: { [leafId]: ptyId } } }, + sleepingAgentSessionsByPaneKey: { + [paneKey]: { worktreeId, automaticResumeBlockedBy: 'legacy-orchestration-worker' } + } + } + const store = { + getWorkspaceSession: vi.fn(() => session), + setWorkspaceSession: vi.fn(), + flushOrThrow: vi.fn() + } + return { + ptyId, + store, + owner: resolveStablePaneOwner( + undefined, + store as unknown as Store, + paneKey, + worktreeId, + connectionId + ) + } +} + +describe('stable-pane fenced attach evidence', () => { + it.each([ + [null, new SessionNotFoundError('worker'), 'exitedBeforeAttach'], + [null, new Error('Session not found: worker'), 'reattachUnverifiable'], + ['host', new SshPtyAbsentFromRelayError('worker'), 'reattachUnverifiable'], + ['host', new SshPtyProvenExitedOnRelayError('worker'), 'exitedBeforeAttach'], + ['host', new Error('socket closed'), 'reattachUnverifiable'] + ] as const)('preserves %s ownership for %s', async (connectionId, error, outcome) => { + const { ptyId, store, owner } = fixture(connectionId) + const spawn = vi.fn(async () => { + throw error + }) + ptyOwnership.set(ptyId, connectionId) + try { + const result = await spawnForStablePane({ + store: store as unknown as Store, + runtime: undefined, + provider: { spawn } as unknown as IPtyProvider, + owner, + worktreeId, + connectionId, + spawnOptions: { cols: 80, rows: 24, paneKey, command: 'must never run' } + }) + expect(result.result).toEqual({ id: ptyId, [outcome]: true }) + expect(spawn).toHaveBeenCalledTimes(1) + expect(spawn).toHaveBeenCalledWith( + expect.objectContaining({ attachOnly: true, command: undefined }) + ) + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + expect(ptyOwnership.get(ptyId)).toBe(connectionId) + expect(store.getWorkspaceSession).toHaveBeenCalledWith(connectionId ? 'ssh:host' : undefined) + } finally { + ptyOwnership.delete(ptyId) + } + }) + it('refuses a newly fenced ownerless pane at the final spawn boundary', async () => { + const { store } = fixture(null) + const spawn = vi.fn(async () => ({ id: 'must-not-spawn' })) + const result = await spawnForStablePane({ + store: store as unknown as Store, + runtime: undefined, + provider: { spawn } as unknown as IPtyProvider, + owner: null, + worktreeId, + spawnOptions: { cols: 80, rows: 24, paneKey } + }) + expect(result.result).toMatchObject({ reattachUnverifiable: true }) + expect(spawn).not.toHaveBeenCalled() + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + }) + it('rechecks a fence committed while attachment is awaiting the host', async () => { + const { store, ptyId } = fixture(null) + const record = store.getWorkspaceSession().sleepingAgentSessionsByPaneKey[paneKey] + delete (record as { automaticResumeBlockedBy?: string }).automaticResumeBlockedBy + const resolveOwner = () => + resolveStablePaneOwner(undefined, store as unknown as Store, paneKey, worktreeId, null) + const owner = resolveOwner() + let calls = 0 + const spawn = vi.fn(async () => { + if (++calls === 1) { + record.automaticResumeBlockedBy = 'legacy-orchestration-worker' + throw new SessionNotFoundError(ptyId) + } + return { id: 'replacement-after-fence' } + }) + const result = await spawnForStablePane({ + runtime: undefined, + store: store as unknown as Store, + provider: { spawn } as unknown as IPtyProvider, + owner, + worktreeId, + connectionId: null, + resolveOwner, + spawnOptions: { cols: 80, rows: 24, paneKey } + }) + expect(result.result).toEqual({ id: ptyId, reattachUnverifiable: true }) + expect(store.setWorkspaceSession).not.toHaveBeenCalled() + expect(spawn).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/ipc/pty/pane/stable-pane-final-fence-recheck.test.ts b/src/main/ipc/pty/pane/stable-pane-final-fence-recheck.test.ts new file mode 100644 index 000000000000..75ce1546fdb0 --- /dev/null +++ b/src/main/ipc/pty/pane/stable-pane-final-fence-recheck.test.ts @@ -0,0 +1,93 @@ +import { expect, it, vi } from 'vitest' +import { getDefaultWorkspaceSession } from '../../../../shared/constants' +import { makePaneKey } from '../../../../shared/stable-pane-id' +import type { Store } from '../../../persistence' +import { SessionNotFoundError } from '../../../daemon/daemon-errors' +import type { IPtyProvider } from '../../../providers/types' +import { resolveStablePaneOwner, spawnForStablePane } from './stable-owner' + +const tabId = 'tab-worker' +const leafId = '5b5b5b5b-5b5b-4b5b-8b5b-5b5b5b5b5b5b' +const paneKey = makePaneKey(tabId, leafId) +const worktreeId = 'folder-worker' + +function fixture(connectionId: string | null) { + const ptyId = connectionId ? `ssh:${connectionId}@@worker` : 'worker' + let session = { + ...getDefaultWorkspaceSession(), + tabsByWorktree: { [worktreeId]: [{ id: tabId, worktreeId, type: 'terminal' }] }, + terminalLayoutsByTabId: { + [tabId]: { + root: { type: 'leaf', leafId }, + activeLeafId: leafId, + ptyIdsByLeafId: { [leafId]: ptyId } + } + }, + sleepingAgentSessionsByPaneKey: { + [paneKey]: { worktreeId, automaticResumeBlockedBy: 'legacy-orchestration-worker' } + } + } + const store = { + getWorkspaceSession: vi.fn(() => session), + setWorkspaceSession: vi.fn((next) => { + session = next + }), + flushOrThrow: vi.fn() + } + return { + ptyId, + store, + owner: resolveStablePaneOwner( + undefined, + store as unknown as Store, + paneKey, + worktreeId, + connectionId + ) + } +} + +it('a fence queued after retirement still stops fresh fallback', async () => { + const { store, ptyId } = fixture(null) + delete ( + store.getWorkspaceSession().sleepingAgentSessionsByPaneKey[paneKey] as { + automaticResumeBlockedBy?: string + } + ).automaticResumeBlockedBy + const resolveOwner = () => { + const owner = resolveStablePaneOwner( + undefined, + store as unknown as Store, + paneKey, + worktreeId, + null + ) + if (!owner) { + queueMicrotask(() => { + store.getWorkspaceSession().sleepingAgentSessionsByPaneKey[ + paneKey + ].automaticResumeBlockedBy = 'legacy-orchestration-worker' + }) + } + return owner + } + const owner = resolveOwner() + const spawn = vi.fn(async (options) => { + if (options.attachOnly) { + throw new SessionNotFoundError(ptyId) + } + return { id: 'replacement-despite-new-fence' } + }) + const result = await spawnForStablePane({ + runtime: undefined, + store: store as unknown as Store, + provider: { spawn } as unknown as IPtyProvider, + owner, + worktreeId, + connectionId: null, + resolveOwner, + spawnOptions: { cols: 80, rows: 24, paneKey } + }) + expect(result.result).toMatchObject({ reattachUnverifiable: true }) + expect(spawn).toHaveBeenCalledTimes(1) +}) diff --git a/src/main/ipc/pty/pane/stable-pane-resume-fence-host.test.ts b/src/main/ipc/pty/pane/stable-pane-resume-fence-host.test.ts new file mode 100644 index 000000000000..a276414ef25d --- /dev/null +++ b/src/main/ipc/pty/pane/stable-pane-resume-fence-host.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it, vi } from 'vitest' +import { isStablePaneResumeBlocked } from './stable-pane-resume-fence' + +describe('review host-scoped fence', () => { + it('reads SSH policy when the identical local pane is not fenced', () => { + const paneKey = 'tab-worker:5b5b5b5b-5b5b-4b5b-8b5b-5b5b5b5b5b5b' + const worktreeId = 'folder-worker' + const local = { sleepingAgentSessionsByPaneKey: {} } + const remote = { + sleepingAgentSessionsByPaneKey: { + [paneKey]: { worktreeId, automaticResumeBlockedBy: 'legacy-orchestration-worker' } + } + } + const store = { getWorkspaceSession: vi.fn((host) => (host === 'ssh:host' ? remote : local)) } + expect(isStablePaneResumeBlocked(store as never, paneKey, worktreeId, 'host')).toBe(true) + }) +}) diff --git a/src/main/ipc/pty/pane/stable-pane-resume-fence.ts b/src/main/ipc/pty/pane/stable-pane-resume-fence.ts new file mode 100644 index 000000000000..2f8f2e512c12 --- /dev/null +++ b/src/main/ipc/pty/pane/stable-pane-resume-fence.ts @@ -0,0 +1,21 @@ +import { toSshExecutionHostId } from '../../../../shared/execution-host' +import { isPaneAutomaticResumeBlocked } from '../../../../shared/terminal-pane-resume-fence' +import type { Store } from '../../../persistence' + +export function isStablePaneResumeBlocked( + store: Store | undefined, + paneKey: string | null | undefined, + worktreeId: string | undefined, + connectionId: string | null | undefined +): boolean { + if (!paneKey || !worktreeId || !store?.getWorkspaceSession) { + return false + } + return isPaneAutomaticResumeBlocked( + store.getWorkspaceSession(connectionId ? toSshExecutionHostId(connectionId) : undefined), + paneKey, + worktreeId + ) +} + +export class StablePaneResumeBlockedError extends Error {} diff --git a/src/main/ipc/pty/provider/liveness.ts b/src/main/ipc/pty/provider/liveness.ts index 8a362b74aaf2..59aa6b1011af 100644 --- a/src/main/ipc/pty/provider/liveness.ts +++ b/src/main/ipc/pty/provider/liveness.ts @@ -1,6 +1,7 @@ import { isRemoteAgentHooksEnabled } from '../../../../shared/agent-hook-relay' import type { AgentSessionOwnerBinding } from '../../../../shared/agent-session-host-authority' import { agentSessionOwnerBindingsEqual } from '../../../../shared/claimed-agent-pty-owner' +import { isSessionNotFoundRefusal } from '../../../../shared/pty-attach-absence-evidence' import { addNodePtyRecoveryHint } from '../../../daemon/node-pty-error-hints' import { SessionNotFoundError } from '../../../daemon/daemon-errors' import type { Store } from '../../../persistence' @@ -64,7 +65,7 @@ export function isPtyAlreadyGoneError(err: unknown): boolean { // Why: the reattach path rewrites the relay's wording to SSH_SESSION_EXPIRED and only this // class preserves that the relay itself answered "absent" rather than the link dropping. isSshPtyAbsentFromRelayError(err) || - /Session not found/i.test(message) + isSessionNotFoundRefusal(message) ) } diff --git a/src/main/ipc/pty/runtime/spawn-early.ts b/src/main/ipc/pty/runtime/spawn-early.ts index 575f89b22e79..cf1dc2f9504a 100644 --- a/src/main/ipc/pty/runtime/spawn-early.ts +++ b/src/main/ipc/pty/runtime/spawn-early.ts @@ -32,6 +32,12 @@ export function adoptMaterializedRuntimePtySpawn( if (startupPromise && !startupAlreadyAwaited) { return startupPromise.then(() => adoptMaterializedRuntimePtySpawn(ctx, true)) } + if ( + ctx.preAdoptedStablePane?.result.exitedBeforeAttach || + ctx.preAdoptedStablePane?.result.reattachUnverifiable + ) { + return ctx.preAdoptedStablePane.result + } if (!ctx.preAdoptedStablePane?.materialized) { return null } diff --git a/src/main/ipc/pty/runtime/spawn-execute.ts b/src/main/ipc/pty/runtime/spawn-execute.ts index 05a9ca82d15b..12439cb78a5b 100644 --- a/src/main/ipc/pty/runtime/spawn-execute.ts +++ b/src/main/ipc/pty/runtime/spawn-execute.ts @@ -1,3 +1,7 @@ +import { + isStablePaneResumeBlocked, + StablePaneResumeBlockedError +} from '../pane/stable-pane-resume-fence' import type { PtySpawnResult } from '../../../providers/types' import { ptyIncarnationById, deletePtyOwnership } from '../provider/ownership-state' import { ptySizes } from '../delivery/visibility-state' @@ -31,15 +35,13 @@ export async function executeRuntimePtySpawn(ctx: RuntimePtySpawnState): Promise } const stablePaneOwnerCandidate = ctx.preAdoptedStablePane ? ctx.preAdoptedStablePane.owner - : args.agentSessionEnsure - ? null - : resolveStablePaneOwner( - ctx.deps.runtime, - ctx.deps.store, - ctx.spawnIdentityPaneKey, - args.worktreeId, - args.connectionId - ) + : resolveStablePaneOwner( + ctx.deps.runtime, + ctx.deps.store, + ctx.spawnIdentityPaneKey, + args.worktreeId, + args.connectionId + ) const expectedPtyId = stablePaneOwnerCandidate?.ptyId ?? ctx.effectiveSessionAppId ?? ctx.sessionId if (expectedPtyId) { @@ -61,7 +63,16 @@ export async function executeRuntimePtySpawn(ctx: RuntimePtySpawnState): Promise throw new Error('client_disconnected') } } - if (args.agentSessionEnsure && !ctx.preAdoptedStablePane) { + if ( + args.agentSessionEnsure && + !ctx.preAdoptedStablePane && + !isStablePaneResumeBlocked( + ctx.deps.store, + ctx.spawnIdentityPaneKey, + args.worktreeId, + args.connectionId + ) + ) { // Why: daemon-backed claims can outlive this controller; import all // proven owners before deciding that an identity is absent. await reconcileAgentSessionOwnerListings() @@ -82,6 +93,16 @@ export async function executeRuntimePtySpawn(ctx: RuntimePtySpawnState): Promise surface: args.agentSessionEnsure.surface, spawn: async () => { assertClientStillConnected() + if ( + isStablePaneResumeBlocked( + ctx.deps.store, + ctx.spawnIdentityPaneKey, + args.worktreeId, + args.connectionId + ) + ) { + throw new StablePaneResumeBlockedError() + } providerResult = await ctx.provider.spawn(ctx.spawnOptions) ctx.rejectedRegistrationCandidate = providerResult // Why: a successful lower-owner return proves physical work committed even if admission sees an early exit. @@ -149,6 +170,9 @@ export async function executeRuntimePtySpawn(ctx: RuntimePtySpawnState): Promise }) ctx.result = stablePaneSpawn.result ctx.stablePaneOwner = stablePaneSpawn.owner + if (ctx.result.exitedBeforeAttach || ctx.result.reattachUnverifiable) { + return + } if ( ctx.stablePaneOwner && ctx.isNewDaemonSession && @@ -196,6 +220,10 @@ export async function executeRuntimePtySpawn(ctx: RuntimePtySpawnState): Promise : ctx.result.wslDistro ) } catch (err) { + if (err instanceof StablePaneResumeBlockedError) { + ctx.result = { id: ctx.sessionId ?? '', reattachUnverifiable: true } + return + } if ( (ctx.isNewDaemonSession || ctx.preparedProvisionalExecutionContext) && ctx.effectiveSessionAppId diff --git a/src/main/ipc/pty/runtime/spawn.ts b/src/main/ipc/pty/runtime/spawn.ts index 2af050cd7963..a8cc44c209ee 100644 --- a/src/main/ipc/pty/runtime/spawn.ts +++ b/src/main/ipc/pty/runtime/spawn.ts @@ -5,8 +5,10 @@ import { makePaneSpawnReservationKey, paneSpawnReservationsByOwnerKey, rejectPaneSpawnReservation, - reservePaneSpawn + reservePaneSpawn, + resolvePaneSpawnReservation } from '../pane/spawn-reservation' +import { isStablePaneResumeBlocked } from '../pane/stable-pane-resume-fence' import { resolveStablePaneOwner } from '../pane/stable-owner' import { ptySizes } from '../delivery/visibility-state' import type { PtyRuntimeControllerDeps } from './controller-deps' @@ -19,6 +21,8 @@ import { createRuntimePtySpawnState, type RuntimePtySpawnArgs } from './spawn-st function toRuntimeSpawnReply(result: { id: string + exitedBeforeAttach?: true + reattachUnverifiable?: true incarnationId?: string wslDistro?: string | null stablePaneOwner?: { handle: string; tabId: string; leafId: string } @@ -26,6 +30,8 @@ function toRuntimeSpawnReply(result: { }) { return { id: result.id, + ...(result.exitedBeforeAttach ? { exitedBeforeAttach: true as const } : {}), + ...(result.reattachUnverifiable ? { reattachUnverifiable: true as const } : {}), ...(result.incarnationId ? { incarnationId: result.incarnationId } : {}), ...(typeof result.wslDistro === 'string' ? { wslDistro: result.wslDistro } : {}), ...(result.stablePaneOwner ? { stablePaneOwner: result.stablePaneOwner } : {}), @@ -71,6 +77,12 @@ export async function spawnPtyFromRuntimeController( args.connectionId ) : null + if ( + !existingOwner && + isStablePaneResumeBlocked(deps.store, paneKey, args.worktreeId, args.connectionId) + ) { + return { id: args.sessionId ?? '', reattachUnverifiable: true as const } + } if (ownerKey && !existingOwner && !paneSpawnReservationsByOwnerKey.has(ownerKey)) { ctx.paneSpawnReservationKey = ownerKey ctx.paneSpawnReservation = reservePaneSpawn(ownerKey) @@ -95,6 +107,18 @@ export async function spawnPtyFromRuntimeController( return toRuntimeSpawnReply(earlyReserved) } await executeRuntimePtySpawn(ctx) + if (ctx.result.exitedBeforeAttach || ctx.result.reattachUnverifiable) { + restoreProvisionalPtySize(ctx) + deps.runtime?.cancelPendingPtyRegistration?.(ctx.pendingRegistrationPtyId ?? ctx.result.id) + ctx.pendingRegistrationPtyId = null + return toRuntimeSpawnReply( + resolvePaneSpawnReservation( + ctx.paneSpawnReservationKey, + ctx.paneSpawnReservation, + ctx.result + ) + ) + } return toRuntimeSpawnReply(await commitRuntimePtySpawn(ctx)) } catch (err) { if (ctx.pendingRegistrationPtyId) { diff --git a/src/main/providers/pty-spawn-result.ts b/src/main/providers/pty-spawn-result.ts index 90b41d9656a6..fec7e6883b1e 100644 --- a/src/main/providers/pty-spawn-result.ts +++ b/src/main/providers/pty-spawn-result.ts @@ -6,6 +6,8 @@ import type { PtySourceReceivingActivation } from '../../shared/pty-source-recei import type { TerminalOwner } from '../../shared/terminal-owner' export type PtySpawnResult = { + exitedBeforeAttach?: true + reattachUnverifiable?: true agentSessionEnsure?: AgentSessionClaimedSpawnResult /** App-facing PTY id. Remote providers must return globally routable ids, * not relay-local handles, because renderer/runtime IPC routes by this key. */ diff --git a/src/main/runtime/orca-runtime-create-terminal.ts b/src/main/runtime/orca-runtime-create-terminal.ts index ec8a3e789416..3245f2914c17 100644 --- a/src/main/runtime/orca-runtime-create-terminal.ts +++ b/src/main/runtime/orca-runtime-create-terminal.ts @@ -4,6 +4,9 @@ import * as dependencies from './orca-runtime-create-terminal-dependencies' import { createDesktopTerminal } from './orca-runtime-create-terminal-desktop' import { buildRuntimeAgentTeamsLaunchPlan } from './orca-runtime-agent-teams-launch-plan' import { createPtySpawnCommitReporter } from './orca-runtime-report-pty-spawn-commit' +import { terminalAttachRefusal } from './terminal-attach-refusal' +import { revealCreatedTerminal } from './terminal-create-reveal' +import { resolveTerminalCreatePaneIdentity } from './terminal-create-pane-identity' export class OrcaRuntimeWithCreateTerminal extends OrcaRuntimeWithTerminalCreateDeduplication { async createTerminal( @@ -33,15 +36,7 @@ export class OrcaRuntimeWithCreateTerminal extends OrcaRuntimeWithTerminalCreate this.resolveWorkspaceTerminalStartupCwd(workspace, launchOpts.cwd) ?? workspace.path let preAllocatedHandle = launchOpts.preAllocatedHandle ?? this.createPreAllocatedTerminalHandle() - const hintedTabId = launchOpts.tabId?.trim() - const canAdoptPaneIdentity = - hintedTabId !== undefined && - dependencies.isValidHostTerminalTabId(hintedTabId) && - launchOpts.leafId !== undefined && - dependencies.isTerminalLeafId(launchOpts.leafId) - let tabId = canAdoptPaneIdentity ? (hintedTabId as string) : dependencies.randomUUID() - let leafId = canAdoptPaneIdentity ? (launchOpts.leafId as string) : dependencies.randomUUID() - let paneKey = dependencies.makePaneKey(tabId, leafId) + let { tabId, leafId, paneKey } = resolveTerminalCreatePaneIdentity(launchOpts) const claimedStablePaneCreate = this.ptyController.claimStablePaneCreate?.({ worktreeId: workspace.id, connectionId: workspace.connectionId, @@ -70,6 +65,16 @@ export class OrcaRuntimeWithCreateTerminal extends OrcaRuntimeWithTerminalCreate tabId, leafId }) + const refusalPane = { tabId, paneKey, worktreeId: workspace.id } + const refusedAdoption = + adoptedBeforeLaunch && + terminalAttachRefusal(adoptedBeforeLaunch.result, { + handle: adoptedBeforeLaunch.owner.handle ?? preAllocatedHandle, + ...refusalPane + }) + if (refusedAdoption) { + return refusedAdoption + } const launchToken = launchOpts.launchConfig ? (launchOpts.launchToken ?? dependencies.randomUUID()) : undefined @@ -175,6 +180,13 @@ export class OrcaRuntimeWithCreateTerminal extends OrcaRuntimeWithTerminalCreate } finally { releaseStablePaneCreate?.() } + const refusedSpawn = terminalAttachRefusal(result, { + handle: result.stablePaneOwner?.handle ?? preAllocatedHandle, + ...refusalPane + }) + if (refusedSpawn) { + return refusedSpawn + } if (!result.stablePaneOwner) { reportPtySpawnCommitted() } @@ -250,32 +262,21 @@ export class OrcaRuntimeWithCreateTerminal extends OrcaRuntimeWithTerminalCreate ...(cwd !== workspace.path ? { startupCwd: cwd } : {}) }) } - let surface: dependencies.RuntimeTerminalCreate['surface'] = 'background' - let warning: string | undefined - if (presentation !== 'background' && this.notifier?.revealTerminalSession) { - try { - await this.notifier.revealTerminalSession(workspace.id, { - ptyId: result.id, - title: launchOpts.title ?? null, - ...(cwd !== workspace.path ? { cwd } : {}), - ...(effectiveLaunchConfig ? { launchConfig: effectiveLaunchConfig } : {}), - ...(launchToken ? { launchToken } : {}), - ...(launchOpts.launchAgent ? { launchAgent: launchOpts.launchAgent } : {}), - ...(launchOpts.viewMode ? { viewMode: launchOpts.viewMode } : {}), - activate: presentation === 'focused', - ...(presentation ? { presentation } : {}), - ...dependencies.ownerSurfacing(opts.surfaceOwner !== false), - tabId, - leafId - }) - surface = 'visible' - } catch (err) { - console.warn(`[terminal-create] failed to create inactive tab for ${result.id}:`, err) - warning = dependencies.createTerminalRevealWarning(handle, err) - } - } else if (presentation !== 'background') { - warning = dependencies.createTerminalRevealWarning(handle) - } + const { surface, warning } = await revealCreatedTerminal({ + notifier: this.notifier, + worktreeId: workspace.id, + workspacePath: workspace.path, + handle, + ptyId: result.id, + tabId, + leafId, + cwd, + presentation, + launchOpts, + launchConfig: effectiveLaunchConfig, + launchToken, + surfaceOwner: opts.surfaceOwner + }) return { handle, tabId, diff --git a/src/main/runtime/orca-runtime-terminal-attach-refusal.test.ts b/src/main/runtime/orca-runtime-terminal-attach-refusal.test.ts new file mode 100644 index 000000000000..0ed77eb42b86 --- /dev/null +++ b/src/main/runtime/orca-runtime-terminal-attach-refusal.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it, vi } from 'vitest' +import { OrcaRuntimeService } from './orca-runtime' + +vi.mock('electron', () => ({ + BrowserWindow: { fromId: vi.fn(() => null) }, + webContents: { fromId: vi.fn(() => null) }, + ipcMain: { + on: vi.fn(), + removeListener: vi.fn() + }, + app: { getPath: vi.fn(() => '/tmp') } +})) + +function stubLaunchScope(runtime: OrcaRuntimeService, path = '/repo/app'): void { + const internals = runtime as unknown as { + resolveTerminalWorkspaceLaunchScope: (selector: string) => Promise<{ + id: string + path: string + connectionId: string | null + repo: null + folderWorkspace: null + }> + } + vi.spyOn(internals, 'resolveTerminalWorkspaceLaunchScope').mockResolvedValue({ + id: 'wt-1', + path, + connectionId: null, + repo: null, + folderWorkspace: null + }) +} + +describe('paired terminal create attach refusal', () => { + it.each(['exitedBeforeAttach', 'reattachUnverifiable'] as const)( + 'returns %s before registering a terminal', + async (outcome) => { + const runtime = new OrcaRuntimeService() + stubLaunchScope(runtime) + const spawn = vi.fn().mockResolvedValue({ id: 'retained-pty', [outcome]: true }) + const registerPty = vi.spyOn(runtime, 'registerPty') + const committed = vi.fn() + runtime.setPtyController({ + spawn, + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + const result = await runtime.createTerminal('id:wt-1', { + onPtySpawnCommitted: committed + }) + expect(result).toMatchObject({ ptyId: 'retained-pty', [outcome]: true }) + expect(registerPty).not.toHaveBeenCalled() + expect(committed).not.toHaveBeenCalled() + } + ) + + it.each(['exitedBeforeAttach', 'reattachUnverifiable'] as const)( + 'returns adopted %s before requesting a spawn', + async (outcome) => { + const runtime = new OrcaRuntimeService() + stubLaunchScope(runtime) + const spawn = vi.fn().mockResolvedValue({ id: 'wrong-pty' }) + const registerPty = vi.spyOn(runtime, 'registerPty') + runtime.setPtyController({ + spawn, + adoptStablePane: vi.fn().mockResolvedValue({ + result: { id: 'retained-pty', [outcome]: true }, + owner: { + handle: 'retained-handle', + tabId: 'tab-1', + leafId: 'pane:1', + ptyId: 'retained-pty' + } + }), + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + const result = await runtime.createTerminal('id:wt-1') + expect(result).toMatchObject({ + handle: 'retained-handle', + ptyId: 'retained-pty', + [outcome]: true + }) + expect(spawn).not.toHaveBeenCalled() + expect(registerPty).not.toHaveBeenCalled() + } + ) +}) diff --git a/src/main/runtime/rpc/methods/agent-session.ts b/src/main/runtime/rpc/methods/agent-session.ts index 08aaa91038e1..77684befee91 100644 --- a/src/main/runtime/rpc/methods/agent-session.ts +++ b/src/main/runtime/rpc/methods/agent-session.ts @@ -1,3 +1,4 @@ +import { negotiateTerminalCreateRefusal } from '../terminal-create-refusal-negotiation' import { z } from 'zod' import { getAgentResumeArgv, @@ -237,20 +238,32 @@ export const AGENT_SESSION_METHODS: RpcAnyMethod[] = [ defineMethod({ name: 'terminal.ensureAgentSession', params: EnsureAgentSessionParams, - handler: (params, { runtime, pairedDeviceId, clientId, clientKind, signal }) => - (runtime as AgentSessionRuntime).ensureAgentSession( - withExecutionHostAgentPresentation(params, clientKind), - callerContext(pairedDeviceId ?? clientId, clientKind, signal) + handler: async ( + params, + { runtime, pairedDeviceId, clientId, clientKind, signal, clientCapabilities } + ) => + negotiateTerminalCreateRefusal( + await (runtime as AgentSessionRuntime).ensureAgentSession( + withExecutionHostAgentPresentation(params, clientKind), + callerContext(pairedDeviceId ?? clientId, clientKind, signal) + ), + clientCapabilities ) }), defineMethod({ name: 'terminal.createAgentSession', params: CreateAgentSessionParams, - handler: (params, { runtime, pairedDeviceId, clientId, clientKind, signal }) => { + handler: async ( + params, + { runtime, pairedDeviceId, clientId, clientKind, signal, clientCapabilities } + ) => { assertOperationTimestampWithinFutureSkew(params.clientOperationId) - return (runtime as AgentSessionRuntime).createAgentSession( - withExecutionHostAgentPresentation(params, clientKind), - callerContext(pairedDeviceId ?? clientId, clientKind, signal) + return negotiateTerminalCreateRefusal( + await (runtime as AgentSessionRuntime).createAgentSession( + withExecutionHostAgentPresentation(params, clientKind), + callerContext(pairedDeviceId ?? clientId, clientKind, signal) + ), + clientCapabilities ) } }) diff --git a/src/main/runtime/rpc/methods/terminal/terminal-lifecycle-methods.ts b/src/main/runtime/rpc/methods/terminal/terminal-lifecycle-methods.ts index 51dde7df4d8e..8da76b3a5d01 100644 --- a/src/main/runtime/rpc/methods/terminal/terminal-lifecycle-methods.ts +++ b/src/main/runtime/rpc/methods/terminal/terminal-lifecycle-methods.ts @@ -1,3 +1,4 @@ +import { negotiateTerminalCreateRefusal } from '../../terminal-create-refusal-negotiation' import { defineMethod, type RpcAnyMethod } from '../../core' import { navigationTargetsHost, @@ -34,7 +35,10 @@ export const TERMINAL_LIFECYCLE_METHODS: RpcAnyMethod[] = [ defineMethod({ name: 'terminal.create', params: TerminalCreateParams, - handler: async (params, { runtime, pairedDeviceId, clientId, clientKind }) => { + handler: async ( + params, + { runtime, pairedDeviceId, clientId, clientKind, clientCapabilities } + ) => { // A focused terminal create predates paired-client navigation. Keep the // authority boundary here so a remote caller cannot activate the host // renderer. This legacy RPC remains a background create for paired viewers; @@ -44,38 +48,41 @@ export const TERMINAL_LIFECYCLE_METHODS: RpcAnyMethod[] = [ const activate = pairedViewer ? false : params.activate === true const presentation = pairedViewer && params.presentation === 'focused' ? 'background' : params.presentation - return { - terminal: await runtime.dedupeTerminalCreate( - pairedDeviceId ?? clientId ?? 'local', - params.worktree, - params.clientMutationId, - params.reconcileExisting === true, - (canonicalWorktreeSelector, preAllocatedHandle) => - runtime.createTerminal(canonicalWorktreeSelector, { - command: params.command, - startupCommandDelivery: params.startupCommandDelivery, - env: params.env, - envToDelete: params.envToDelete, - ...(params.launchConfig ? { launchConfig: params.launchConfig } : {}), - ...(params.resumeProviderSession - ? { resumeProviderSession: params.resumeProviderSession } - : {}), - ...(params.launchToken ? { launchToken: params.launchToken } : {}), - ...(params.launchAgent ? { launchAgent: params.launchAgent } : {}), - ...(params.terminalColorQueryReplies - ? { terminalColorQueryReplies: params.terminalColorQueryReplies } - : {}), - title: params.title, - focus, - rendererBacked: params.rendererBacked === true, - activate, - presentation, - tabId: params.tabId, - leafId: params.leafId, - ...(preAllocatedHandle ? { preAllocatedHandle } : {}) - }) - ) - } + return negotiateTerminalCreateRefusal( + { + terminal: await runtime.dedupeTerminalCreate( + pairedDeviceId ?? clientId ?? 'local', + params.worktree, + params.clientMutationId, + params.reconcileExisting === true, + (canonicalWorktreeSelector, preAllocatedHandle) => + runtime.createTerminal(canonicalWorktreeSelector, { + command: params.command, + startupCommandDelivery: params.startupCommandDelivery, + env: params.env, + envToDelete: params.envToDelete, + ...(params.launchConfig ? { launchConfig: params.launchConfig } : {}), + ...(params.resumeProviderSession + ? { resumeProviderSession: params.resumeProviderSession } + : {}), + ...(params.launchToken ? { launchToken: params.launchToken } : {}), + ...(params.launchAgent ? { launchAgent: params.launchAgent } : {}), + ...(params.terminalColorQueryReplies + ? { terminalColorQueryReplies: params.terminalColorQueryReplies } + : {}), + title: params.title, + focus, + rendererBacked: params.rendererBacked === true, + activate, + presentation, + tabId: params.tabId, + leafId: params.leafId, + ...(preAllocatedHandle ? { preAllocatedHandle } : {}) + }) + ) + }, + clientCapabilities + ) } }), defineMethod({ diff --git a/src/main/runtime/rpc/terminal-create-refusal-negotiation.test.ts b/src/main/runtime/rpc/terminal-create-refusal-negotiation.test.ts new file mode 100644 index 000000000000..0b227f743036 --- /dev/null +++ b/src/main/runtime/rpc/terminal-create-refusal-negotiation.test.ts @@ -0,0 +1,64 @@ +import type { RpcMethod } from './core' +import { expect, it, vi } from 'vitest' +import { TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' +import { terminalAttachRefusal } from '../terminal-attach-refusal' +import { AGENT_SESSION_METHODS } from './methods/agent-session' +import { TERMINAL_LIFECYCLE_METHODS } from './methods/terminal/terminal-lifecycle-methods' +import { mapRuntimeError } from './errors' + +it.each(['terminal.create', 'terminal.createAgentSession', 'terminal.ensureAgentSession'])( + '%s negotiates both refusal outcomes without changing ordinary success', + async (method) => { + const definition = [...TERMINAL_LIFECYCLE_METHODS, ...AGENT_SESSION_METHODS].find( + (entry) => entry.name === method + )! as RpcMethod + for (const outcome of ['exitedBeforeAttach', 'reattachUnverifiable', 'success'] as const) { + const owner = { + handle: 'retained', + tabId: 'tab-1', + paneKey: 'tab-1:pane', + worktreeId: 'wt-1' + } + const terminal = + outcome === 'success' + ? { ...owner, ptyId: 'worker', title: null } + : terminalAttachRefusal({ id: 'worker', [outcome]: true }, owner)! + const result = { terminal, disposition: 'created' } + const runtime = { + createTerminal: vi.fn(async () => terminal), + dedupeTerminalCreate: vi.fn(async () => terminal), + createAgentSession: vi.fn(async () => result), + ensureAgentSession: vi.fn(async () => result) + } + for (const clientCapabilities of [ + undefined, + [], + [TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY] + ]) { + const call = definition.handler( + { + worktree: 'wt-1', + agent: 'codex', + kind: 'explicit', + providerSession: { key: 'session_id', id: 'session-1' }, + clientOperationId: '1752883200000-0123456789abcdef0123456789abcdef' + }, + { runtime, clientCapabilities } as never + ) + if (outcome === 'success' || clientCapabilities?.length) { + await expect(call).resolves.toMatchObject({ terminal }) + } else { + await expect(call).rejects.toMatchObject({ code: 'remote_runtime_unavailable' }) + try { + await call + } catch (error) { + expect(mapRuntimeError('request', { runtimeId: 'host' }, error)).toMatchObject({ + ok: false, + error: { code: 'remote_runtime_unavailable' } + }) + } + } + } + } + } +) diff --git a/src/main/runtime/rpc/terminal-create-refusal-negotiation.ts b/src/main/runtime/rpc/terminal-create-refusal-negotiation.ts new file mode 100644 index 000000000000..4a29c52164eb --- /dev/null +++ b/src/main/runtime/rpc/terminal-create-refusal-negotiation.ts @@ -0,0 +1,19 @@ +import { TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY } from '../../../shared/protocol-version' +import type { RuntimeTerminalCreate } from '../../../shared/runtime-terminal-contracts' +import type { RpcContext } from './core' + +export function negotiateTerminalCreateRefusal( + result: T, + clientCapabilities: RpcContext['clientCapabilities'] +): T { + if ( + (result.terminal.exitedBeforeAttach || result.terminal.reattachUnverifiable) && + !clientCapabilities?.includes(TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY) + ) { + // Older clients interpret every successful create as permission to publish a fresh binding. + throw Object.assign(new Error('Remote terminal attachment is temporarily unavailable.'), { + code: 'remote_runtime_unavailable' + }) + } + return result +} diff --git a/src/main/runtime/runtime-pty-controller-contract.ts b/src/main/runtime/runtime-pty-controller-contract.ts index 73a75af017ed..5845db9dd475 100644 --- a/src/main/runtime/runtime-pty-controller-contract.ts +++ b/src/main/runtime/runtime-pty-controller-contract.ts @@ -82,6 +82,8 @@ export type RuntimePtyController = { } }): Promise<{ id: string + exitedBeforeAttach?: true + reattachUnverifiable?: true pid?: number | null incarnationId?: PtyIncarnationId wslDistro?: string diff --git a/src/main/runtime/terminal-attach-refusal.ts b/src/main/runtime/terminal-attach-refusal.ts new file mode 100644 index 000000000000..d21b26630c86 --- /dev/null +++ b/src/main/runtime/terminal-attach-refusal.ts @@ -0,0 +1,20 @@ +import type { RuntimeTerminalCreate } from '../../shared/runtime-terminal-contracts' + +export function terminalAttachRefusal( + result: { + id: string + exitedBeforeAttach?: true + reattachUnverifiable?: true + }, + owner: { handle: string; tabId: string; paneKey: string; worktreeId: string } +): RuntimeTerminalCreate | null { + if (!result.exitedBeforeAttach && !result.reattachUnverifiable) { + return null + } + return { + ...owner, + ptyId: result.id, + title: null, + ...(result.exitedBeforeAttach ? { exitedBeforeAttach: true } : { reattachUnverifiable: true }) + } +} diff --git a/src/main/runtime/terminal-create-pane-identity.ts b/src/main/runtime/terminal-create-pane-identity.ts new file mode 100644 index 000000000000..98d0f7fb78df --- /dev/null +++ b/src/main/runtime/terminal-create-pane-identity.ts @@ -0,0 +1,19 @@ +import { randomUUID } from 'node:crypto' +import { isValidHostTerminalTabId } from '../../shared/terminal-tab-id' +import { isTerminalLeafId, makePaneKey } from '../../shared/stable-pane-id' + +export function resolveTerminalCreatePaneIdentity(options: { tabId?: string; leafId?: string }): { + tabId: string + leafId: string + paneKey: string +} { + const hintedTabId = options.tabId?.trim() + const canAdopt = + hintedTabId !== undefined && + isValidHostTerminalTabId(hintedTabId) && + options.leafId !== undefined && + isTerminalLeafId(options.leafId) + const tabId = canAdopt ? hintedTabId! : randomUUID() + const leafId = canAdopt ? options.leafId! : randomUUID() + return { tabId, leafId, paneKey: makePaneKey(tabId, leafId) } +} diff --git a/src/main/runtime/terminal-create-reveal.ts b/src/main/runtime/terminal-create-reveal.ts new file mode 100644 index 000000000000..87f270a42502 --- /dev/null +++ b/src/main/runtime/terminal-create-reveal.ts @@ -0,0 +1,48 @@ +import type { RuntimeTerminalCreate, RuntimeTerminalPresentation } from '../../shared/runtime-types' +import type { SleepingAgentLaunchConfig } from '../../shared/agent-session-resume' +import type { RuntimeNotifier } from './runtime-notifier-contract' +import type { TerminalCreateOptions } from './runtime-terminal-contracts' +import { createTerminalRevealWarning, ownerSurfacing } from './orca-runtime-core' + +export async function revealCreatedTerminal(args: { + notifier?: RuntimeNotifier | null + worktreeId: string + workspacePath: string + handle: string + ptyId: string + tabId: string + leafId: string + cwd: string + presentation: RuntimeTerminalPresentation | undefined + launchOpts: TerminalCreateOptions + launchConfig?: SleepingAgentLaunchConfig | null + launchToken?: string + surfaceOwner?: boolean +}): Promise> { + if (args.presentation === 'background') { + return { surface: 'background' } + } + if (!args.notifier?.revealTerminalSession) { + return { surface: 'background', warning: createTerminalRevealWarning(args.handle) } + } + try { + await args.notifier.revealTerminalSession(args.worktreeId, { + ptyId: args.ptyId, + title: args.launchOpts.title ?? null, + ...(args.cwd !== args.workspacePath ? { cwd: args.cwd } : {}), + ...(args.launchConfig ? { launchConfig: args.launchConfig } : {}), + ...(args.launchToken ? { launchToken: args.launchToken } : {}), + ...(args.launchOpts.launchAgent ? { launchAgent: args.launchOpts.launchAgent } : {}), + ...(args.launchOpts.viewMode ? { viewMode: args.launchOpts.viewMode } : {}), + activate: args.presentation === 'focused', + ...(args.presentation ? { presentation: args.presentation } : {}), + ...ownerSurfacing(args.surfaceOwner !== false), + tabId: args.tabId, + leafId: args.leafId + }) + return { surface: 'visible' } + } catch (error) { + console.warn(`[terminal-create] failed to create inactive tab for ${args.ptyId}:`, error) + return { surface: 'background', warning: createTerminalRevealWarning(args.handle, error) } + } +} diff --git a/src/preload/api/pty-api.ts b/src/preload/api/pty-api.ts index d2d547d98cb7..594a850b7c16 100644 --- a/src/preload/api/pty-api.ts +++ b/src/preload/api/pty-api.ts @@ -65,6 +65,8 @@ export type PtyApi = { isReattach?: boolean isAlternateScreen?: boolean replay?: string + exitedBeforeAttach?: boolean + reattachUnverifiable?: boolean sessionExpired?: boolean coldRestore?: { scrollback: string; cwd: string; cols?: number; rows?: number } startupCwdFallback?: { kind: 'worktree'; cwd: string } diff --git a/src/renderer/src/components/terminal-pane/ipc-pty-attach-evidence.test.ts b/src/renderer/src/components/terminal-pane/ipc-pty-attach-evidence.test.ts new file mode 100644 index 000000000000..e5ae48022ea8 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/ipc-pty-attach-evidence.test.ts @@ -0,0 +1,49 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { installIpcPtyWindow, restorePtySpecWindow } from './pty-transport-test-harness' + +describe('IPC attach evidence', () => { + const originalWindow = globalThis.window + beforeEach(() => { + vi.resetModules() + installIpcPtyWindow(originalWindow, {}) + }) + afterEach(() => restorePtySpecWindow(originalWindow)) + + it.each(['exitedBeforeAttach', 'reattachUnverifiable'] as const)( + 'preserves main %s without publishing a successful spawn or attach', + async (evidence) => { + const { createIpcPtyTransport } = await import('./pty-transport') + const id = 'ssh:ssh-1@@old-epoch-pty' + const outcome = { id, [evidence]: true } + vi.mocked(window.api.pty.spawn).mockResolvedValueOnce(outcome) + const onPtySpawn = vi.fn() + const onConnect = vi.fn() + const transport = createIpcPtyTransport({ connectionId: 'ssh-1', onPtySpawn }) + expect(await transport.connect({ url: '', sessionId: id, callbacks: { onConnect } })).toEqual( + outcome + ) + expect(onPtySpawn).not.toHaveBeenCalled() + expect(onConnect).not.toHaveBeenCalled() + expect(transport.getPtyId()).toBeNull() + transport.destroy?.() + expect(window.api.pty.kill).not.toHaveBeenCalled() + } + ) + + it.each([ + 'SSH_SESSION_EXPIRED: old-epoch-pty', + 'Session not found: old-epoch-pty', + 'SSH relay connection unavailable' + ])('does not reconstruct observed exit from %s', async (message) => { + const { createIpcPtyTransport } = await import('./pty-transport') + vi.mocked(window.api.pty.spawn).mockRejectedValueOnce(new Error(message)) + const transport = createIpcPtyTransport({ connectionId: 'ssh-1' }) + const result = await transport.connect({ + url: '', + sessionId: 'ssh:ssh-1@@old-epoch-pty', + callbacks: {} + }) + expect(result && typeof result === 'object' && result.exitedBeforeAttach).toBeFalsy() + transport.destroy?.() + }) +}) diff --git a/src/renderer/src/components/terminal-pane/ipc-pty-connect.ts b/src/renderer/src/components/terminal-pane/ipc-pty-connect.ts index 3023236de0ef..db615f4856e8 100644 --- a/src/renderer/src/components/terminal-pane/ipc-pty-connect.ts +++ b/src/renderer/src/components/terminal-pane/ipc-pty-connect.ts @@ -88,6 +88,9 @@ export async function connectIpcPty( // recorded before we asked for a PTY, so it belongs to that earlier owner, not to us. const priorIncarnationFence = currentPreHandlerPtySequence() const spawnResult = await spawnIpcPty(transportOptions, options, admittedSessionId) + if (spawnResult.exitedBeforeAttach || spawnResult.reattachUnverifiable) { + return context.isDestroyed() ? undefined : spawnResult + } const retireFreshSpawn = async (): Promise => { // A newer generation may already own a recycled id; an id-only kill would retire its PTY. if ( diff --git a/src/renderer/src/components/terminal-pane/pr-base-pty-transport-test-loader.ts b/src/renderer/src/components/terminal-pane/pr-base-pty-transport-test-loader.ts new file mode 100644 index 000000000000..11f5611b2a96 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/pr-base-pty-transport-test-loader.ts @@ -0,0 +1,33 @@ +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { transformSync } from 'esbuild' +import type { createRemoteRuntimePtyTransport } from './remote-runtime-pty-transport' + +export async function loadPrBasePtyTransport(): Promise<{ + createRemoteRuntimePtyTransport: typeof createRemoteRuntimePtyTransport +}> { + // Frozen PR-base reader, evaluated with the same mocked dependencies as the current transport. + const source = readFileSync( + resolve('tests/fixtures/terminal/pr-base-1d1b73c40850-pty-transport.txt'), + 'utf8' + ) + const { code: outputText } = transformSync(source, { loader: 'ts', format: 'cjs' }) + const dependencies = new Map() + for (const match of outputText.matchAll(/require\("([^"]+)"\)/g)) { + const specifier = match[1] + if (!dependencies.has(specifier)) { + const target = specifier.startsWith('@/') + ? resolve('src/renderer/src', specifier.slice(2)) + : resolve(__dirname, specifier) + dependencies.set(specifier, await import(/* @vite-ignore */ target)) + } + } + const exports = {} as { createRemoteRuntimePtyTransport: typeof createRemoteRuntimePtyTransport } + const module = { exports } + new Function('require', 'exports', 'module', outputText)( + (name: string) => dependencies.get(name), + exports, + module + ) + return module.exports +} diff --git a/src/renderer/src/components/terminal-pane/pty-connection-agent-session-resume.test.ts b/src/renderer/src/components/terminal-pane/pty-connection-agent-session-resume.test.ts index c93294ba2f9b..680590dc6620 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection-agent-session-resume.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection-agent-session-resume.test.ts @@ -406,84 +406,94 @@ describe('connectPanePty', () => { expect(mockStoreState.clearSleepingAgentSession).not.toHaveBeenCalled() }) - it('does not resume a live provider session while legacy worker recovery owns the pane', async () => { - const { connectPanePty } = await import('./pty-connection') - const retainedPtyId = 'wt-1@@lost-pty' - const transport = createMockTransport() - transport.connect.mockImplementation(async ({ sessionId }: { sessionId?: string }) => - sessionId - ? { - id: 'fresh-pty', - coldRestore: { scrollback: 'cold-payload', cwd: '/tmp/wt-1' } - } - : 'fresh-pty' - ) - transportFactoryQueue.push(transport) - const paneKey = makePaneKey('tab-1', LEAF_1) - mockStoreState = { - ...mockStoreState, - tabsByWorktree: { - 'wt-1': [{ id: 'tab-1', ptyId: retainedPtyId }] - }, - settings: { - ...mockStoreState.settings, - agentCmdOverrides: {} - }, - agentStatusByPaneKey: { - [paneKey]: { - paneKey, - state: 'working', - prompt: 'finish the task', - agentType: 'codex', - providerSession: { key: 'session_id', id: 'codex-session-1' } + it.each(['live', 'undefined', 'thrown', 'unverifiable'] as const)( + 'reattaches a fenced local worker without retiring its binding on %s', + async (outcome) => { + const { connectPanePty } = await import('./pty-connection') + const retainedPtyId = 'wt-1@@lost-pty' + const transport = createMockTransport() + transport.connect.mockImplementation(async ({ sessionId }: { sessionId?: string }) => { + if (outcome === 'undefined') { + return undefined } - }, - sleepingAgentSessionsByPaneKey: { - [paneKey]: { - paneKey, - tabId: 'tab-1', - worktreeId: 'wt-1', - agent: 'codex', - providerSession: { key: 'session_id', id: 'codex-session-1' }, - prompt: 'finish the task', - state: 'working', - capturedAt: 1, - updatedAt: 1, - automaticResumeBlockedBy: 'legacy-orchestration-worker' + if (outcome === 'thrown') { + throw new Error('daemon unavailable') } - } - } as StoreState + if (outcome === 'unverifiable') { + return { id: retainedPtyId, reattachUnverifiable: true } + } + return sessionId ? { id: sessionId, isReattach: true } : 'fresh-pty' + }) + transportFactoryQueue.push(transport) + const paneKey = makePaneKey('tab-1', LEAF_1) + mockStoreState = { + ...mockStoreState, + tabsByWorktree: { + 'wt-1': [{ id: 'tab-1', ptyId: retainedPtyId }] + }, + settings: { + ...mockStoreState.settings, + agentCmdOverrides: {} + }, + agentStatusByPaneKey: { + [paneKey]: { + paneKey, + state: 'working', + prompt: 'finish the task', + agentType: 'codex', + providerSession: { key: 'session_id', id: 'codex-session-1' } + } + }, + sleepingAgentSessionsByPaneKey: { + [paneKey]: { + paneKey, + tabId: 'tab-1', + worktreeId: 'wt-1', + agent: 'codex', + providerSession: { key: 'session_id', id: 'codex-session-1' }, + prompt: 'finish the task', + state: 'working', + capturedAt: 1, + updatedAt: 1, + automaticResumeBlockedBy: 'legacy-orchestration-worker' + } + } + } as StoreState - connectPanePty( - createPane(1) as never, - createManager(1) as never, - createDeps({ + const deps = createDeps({ restoredLeafId: LEAF_1, restoredPtyIdByLeafId: { [LEAF_1]: retainedPtyId } - }) as never - ) - await flushAsyncTicks(20) - await new Promise((resolve) => setTimeout(resolve, 70)) + }) + const binding = connectPanePty( + createPane(1) as never, + createManager(1) as never, + deps as never + ) + await flushAsyncTicks(20) + await new Promise((resolve) => setTimeout(resolve, 70)) - expect(transport.connect).not.toHaveBeenCalled() - expect(transport.attach).toHaveBeenCalledWith( - expect.objectContaining({ existingPtyId: retainedPtyId }) - ) - const attachOptions = transport.attach.mock.calls[0]?.[0] as Record - expect(attachOptions).not.toHaveProperty('cols') - expect(attachOptions).not.toHaveProperty('rows') - expect(mockStoreState.registerAgentLaunchConfig).not.toHaveBeenCalled() - expect(mockStoreState.clearSleepingAgentSession).not.toHaveBeenCalled() - }) + expect(deps.clearTabPtyId).not.toHaveBeenCalled() + expect(deps.clearExitedPanePtyLayoutBinding).not.toHaveBeenCalled() + expect(transport.connect).toHaveBeenCalledTimes(1) + expect(transport.attach).not.toHaveBeenCalled() + const connectOptions = transport.connect.mock.calls[0]?.[0] as Record + expect(connectOptions).toMatchObject({ sessionId: retainedPtyId }) + expect(connectOptions).not.toHaveProperty('attachOnly') + expect(connectOptions).not.toHaveProperty('command') + expect(connectOptions).not.toHaveProperty('launchConfig') + expect(connectOptions).not.toHaveProperty('resumeProviderSession') + expect(mockStoreState.registerAgentLaunchConfig).not.toHaveBeenCalled() + expect(mockStoreState.clearSleepingAgentSession).not.toHaveBeenCalled() + binding.dispose() + } + ) it('does not replace a missing retained legacy worker over direct SSH', async () => { const { connectPanePty } = await import('./pty-connection') const retainedPtyId = toAppSshPtyId('ssh-a', 'missing-legacy-worker') const transport = createMockTransport() transport.getConnectionId.mockReturnValue('ssh-a') - transport.attach.mockImplementation(() => { - throw new Error('remote PTY missing') - }) + transport.connect.mockRejectedValue(new Error('remote PTY missing')) transportFactoryQueue.push(transport) const paneKey = makePaneKey('tab-1', LEAF_1) mockStoreState = { @@ -517,10 +527,12 @@ describe('connectPanePty', () => { await flushAsyncTicks(20) await new Promise((resolve) => setTimeout(resolve, 70)) - expect(transport.attach).toHaveBeenCalledWith( - expect.objectContaining({ existingPtyId: retainedPtyId }) - ) - expect(transport.connect).not.toHaveBeenCalled() + expect(transport.attach).not.toHaveBeenCalled() + expect(transport.connect.mock.calls[0]?.[0]).toMatchObject({ + sessionId: retainedPtyId + }) + // An unproven failure is `unverifiable`: the pane keeps its binding rather than retiring a + // session it may never replace (docs/reference/ssh-execution-boundary.md). expect(deps.clearTabPtyId).not.toHaveBeenCalled() expect(mockStoreState.registerAgentLaunchConfig).not.toHaveBeenCalled() }) @@ -530,9 +542,7 @@ describe('connectPanePty', () => { const retainedPtyId = toAppSshPtyId('ssh-a', 'missing-legacy-worker') const transport = createMockTransport() transport.getConnectionId.mockReturnValue('ssh-a') - transport.attach.mockImplementation(() => { - throw new Error('remote PTY missing') - }) + transport.connect.mockRejectedValue(new Error('remote PTY missing')) transportFactoryQueue.push(transport) const paneKey = makePaneKey('tab-1', LEAF_1) mockStoreState = { @@ -569,11 +579,11 @@ describe('connectPanePty', () => { await new Promise((resolve) => setTimeout(resolve, 70)) expect(window.api.ssh.connect).toHaveBeenCalledWith({ targetId: 'ssh-a' }) - expect(transport.attach).toHaveBeenCalledWith( - expect.objectContaining({ existingPtyId: retainedPtyId }) - ) - expect(transport.connect).not.toHaveBeenCalled() - expect(mockStoreState.removeDeferredSshSessionId).not.toHaveBeenCalled() + expect(transport.attach).not.toHaveBeenCalled() + // Same deferred-SSH flow as every other restored pane. + expect(transport.connect.mock.calls[0]?.[0]).toMatchObject({ + sessionId: retainedPtyId + }) expect(deps.clearTabPtyId).not.toHaveBeenCalled() expect(mockStoreState.registerAgentLaunchConfig).not.toHaveBeenCalled() }) diff --git a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-attach.ts b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-attach.ts index 1b05fa09e603..8f178165193d 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-attach.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-attach.ts @@ -1,4 +1,3 @@ -import { scheduleRuntimeGraphSync } from '@/runtime/sync-runtime-graph' import { useAppStore } from '@/store' import { isRuntimeOwnedSshTargetId } from '../../../../../shared/execution-host' import { resolveSshPaneConnectGate } from '../ssh-pane-connect-gate' @@ -11,7 +10,7 @@ import { toProcessExitStartup } from './process-exit-startup' import type { ConnectPanePtySession } from './connect-pane-pty-session' import { runDeferredSessionReattachChoice } from './deferred-session-reattach-choice' -import { recoverUnverifiableDirectSshReattach } from './direct-ssh-reattach-recovery' +import { recoverUnverifiableReattach } from './unverifiable-reattach-recovery' export function runDeferredSessionAttach(session: ConnectPanePtySession): void { const isCurrentPaneTransport = (): boolean => @@ -61,8 +60,7 @@ export function runDeferredSessionAttach(session: ConnectPanePtySession): void { console.warn( `[pty-connection] SSH tab=${session.deps.tabId} connectionId=${session.connectionId} pendingSessionId=${pendingSessionId} sshConnected=${gate.sshConnected}` ) - const legacyWorkerOwnsPane = session.isLegacyWorkerAutomaticResumeBlocked() - if (gate.enterDeferredFlow && (!legacyWorkerOwnsPane || !gate.sshConnected)) { + if (gate.enterDeferredFlow) { // Paint main's parked model while SSH recovery continues off the render path. session.prepaintParkedSshSnapshot(pendingSessionId) void (async () => { @@ -115,13 +113,6 @@ export function runDeferredSessionAttach(session: ConnectPanePtySession): void { } useAppStore.getState().removeDeferredSshReconnectTarget(session.connectionId) if (pendingSessionId) { - if (session.isLegacyWorkerAutomaticResumeBlocked()) { - if (session.attachRetainedLegacyPty(pendingSessionId)) { - useAppStore.getState().removeDeferredSshSessionId(session.deps.tabId) - scheduleRuntimeGraphSync() - } - return - } console.warn( `[pty-connection] Attempting reattach for tab=${session.deps.tabId} sessionId=${pendingSessionId}` ) @@ -294,7 +285,7 @@ export function runDeferredSessionAttach(session: ConnectPanePtySession): void { return } session.reportError(err instanceof Error ? err.message : String(err)) - recoverUnverifiableDirectSshReattach(session, pendingSessionId) + recoverUnverifiableReattach(session, pendingSessionId) }) session.armDirectSshPaneRetryTimeout( trackedReattachPromise, diff --git a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-choice.test.ts b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-choice.test.ts new file mode 100644 index 000000000000..7f178c6b099c --- /dev/null +++ b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-choice.test.ts @@ -0,0 +1,47 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { runDeferredSessionReattachChoice } from './deferred-session-reattach-choice' + +vi.mock('@/runtime/sync-runtime-graph', () => ({ scheduleRuntimeGraphSync: vi.fn() })) +vi.mock('@/store', () => ({ + useAppStore: { getState: () => ({ tabsByWorktree: {}, ptyIdsByTabId: {} }) } +})) +vi.mock('../pty-dispatcher', () => ({ getEagerPtyBufferHandle: vi.fn() })) + +describe('fenced sleeping paired pane choice', () => { + afterEach(() => vi.clearAllMocks()) + it('attaches the retained remote leaf without clearing its binding or resuming', () => { + const id = 'remote:env-1@@terminal-1' + const session = { + deps: { + worktreeId: 'wt-1', + tabId: 'tab-1', + restoredLeafId: 'leaf-1', + restoredPtyIdByLeafId: { 'leaf-1': id }, + paneTransportsRef: { current: new Map() }, + clearTabPtyId: vi.fn() + }, + pane: { id: 1 }, + getSleepingRecordForPane: () => ({}), + isLegacyWorkerAutomaticResumeBlocked: () => true, + buildColdRestoreAgentResumeStartup: vi.fn(), + syncPanePtyLayoutBinding: vi.fn(), + clearPaneMode2031State: vi.fn(), + clearHiddenOutputRestoreState: vi.fn(), + captureTransportOutputCallbacks: () => ({ callbacks: {} }), + transport: { attach: vi.fn(), getPtyId: () => id }, + bindActivePanePty: vi.fn(), + registerPaneSerializerFor: vi.fn(), + reportError: vi.fn(), + startFreshSpawn: vi.fn(), + startFreshColdRestoreAgentResume: vi.fn() + } + runDeferredSessionReattachChoice(session as never) + expect(session.transport.attach).toHaveBeenCalledWith( + expect.objectContaining({ existingPtyId: id }) + ) + expect(session.deps.clearTabPtyId).not.toHaveBeenCalled() + expect(session.syncPanePtyLayoutBinding).not.toHaveBeenCalled() + expect(session.startFreshSpawn).not.toHaveBeenCalled() + expect(session.startFreshColdRestoreAgentResume).not.toHaveBeenCalled() + }) +}) diff --git a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-choice.ts b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-choice.ts index bb4eab2444c0..3082a609e766 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-choice.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-choice.ts @@ -27,7 +27,11 @@ export function runDeferredSessionReattachChoice(session: ConnectPanePtySession) const existingPtyId = storeSnapshot.tabsByWorktree[session.deps.worktreeId]?.find( (t) => t.id === session.deps.tabId )?.ptyId - const hasSleepingAgentSession = Boolean(session.getSleepingRecordForPane(storeSnapshot)) + // Why: the legacy-worker fence forbids resuming this pane's provider session, so its sleeping + // record must not steer the pane into any cold-restore or wake branch. It still reattaches. + const hasResumableSleepingAgentSession = + Boolean(session.getSleepingRecordForPane(storeSnapshot)) && + !session.isLegacyWorkerAutomaticResumeBlocked() // Why: the tab-level fallback must not steal a PTY a setup sibling already published while the main pane waited for split geometry. const tabFallbackPtyId = @@ -41,7 +45,7 @@ export function runDeferredSessionReattachChoice(session: ConnectPanePtySession) const restoredSessionId = restoredPtyId ?? null const sleptRemoteRuntimeSessionId = - restoredSessionId && isRemoteRuntimePtyId(restoredSessionId) && hasSleepingAgentSession + restoredSessionId && isRemoteRuntimePtyId(restoredSessionId) && hasResumableSleepingAgentSession ? restoredSessionId : null const detachedLivePtyId = @@ -53,7 +57,9 @@ export function runDeferredSessionReattachChoice(session: ConnectPanePtySession) : tabFallbackPtyId : null const detachedRemoteLeafPtyId = - restoredSessionId && isRemoteRuntimePtyId(restoredSessionId) && !hasSleepingAgentSession + restoredSessionId && + isRemoteRuntimePtyId(restoredSessionId) && + !hasResumableSleepingAgentSession ? restoredSessionId : null const candidateReattachSessionId = @@ -87,11 +93,6 @@ export function runDeferredSessionReattachChoice(session: ConnectPanePtySession) currentTabLivePtyIds.includes(candidateReattachSessionId) ? candidateReattachSessionId : null - // Why: after a daemon crash + cold restore, a stale session-to-tab mapping can make a tab hold a ptyId from another worktree. - // Restoring it would paint the wrong terminal content, so drop the reattach and spawn fresh. - const legacyAttachOnlyPtyId = session.isLegacyWorkerAutomaticResumeBlocked() - ? candidateReattachSessionId - : null const pairedParkedReattachSessionId = session.mountFollowsTerminalPark && candidateReattachSessionId && @@ -99,64 +100,51 @@ export function runDeferredSessionReattachChoice(session: ConnectPanePtySession) canRestorePairedParkedTerminal(candidateReattachSessionId) ? candidateReattachSessionId : null - const deferredReattachSessionId = legacyAttachOnlyPtyId - ? null - : (runtimeHostPtyWakeHint ?? - pairedParkedReattachSessionId ?? - (candidateReattachSessionId && - !isRemoteRuntimePtyId(candidateReattachSessionId) && - !candidateHasEagerBuffer && - isSessionOwnedByWorktree(candidateReattachSessionId, session.deps.worktreeId) - ? candidateReattachSessionId - : null)) + const deferredReattachSessionId = + runtimeHostPtyWakeHint ?? + pairedParkedReattachSessionId ?? + (candidateReattachSessionId && + !isRemoteRuntimePtyId(candidateReattachSessionId) && + !candidateHasEagerBuffer && + isSessionOwnedByWorktree(candidateReattachSessionId, session.deps.worktreeId) + ? candidateReattachSessionId + : null) recordPtyConnectDiagnostic( `pane=${session.pane.id} tab=${session.deps.tabId} restored=${restoredPtyId} existing=${existingPtyId} detached=${detachedRemoteLeafPtyId ?? detachedLivePtyId} reattach=${deferredReattachSessionId} hasTransport=${session.hadExistingPaneTransportAtConnect} pendingKey=${session.pendingSpawnKey}` ) if (deferredReattachSessionId) { startDeferredSessionReattach(session, deferredReattachSessionId) - } else if ( - legacyAttachOnlyPtyId || - detachedRemoteLeafPtyId || - detachedLivePtyId || - eagerLivePtyId - ) { + } else if (detachedRemoteLeafPtyId || detachedLivePtyId || eagerLivePtyId) { // Why: mirrored web-leaf panes must attach to their exact remote PTY, not spawn a replacement host tab. // eagerLivePtyId covers a still-live background PTY (e.g. an automation agent) with a live eager buffer to adopt. - const attachPtyId = - legacyAttachOnlyPtyId ?? detachedRemoteLeafPtyId ?? detachedLivePtyId ?? eagerLivePtyId! + const attachPtyId = detachedRemoteLeafPtyId ?? detachedLivePtyId ?? eagerLivePtyId! recordPtyConnectDiagnostic(`pane=${session.pane.id} -> ATTACH detached=${attachPtyId}`) session.allowInitialIdleCacheSeed = false - if (legacyAttachOnlyPtyId) { - if (session.attachRetainedLegacyPty(legacyAttachOnlyPtyId) && session.connectionId) { - useAppStore.getState().removeDeferredSshSessionId(session.deps.tabId) - } - } else { - // Why: surface synchronous attach failures via session.reportError so the pane shows a diagnostic instead of a blank surface. - // On throw, clear the stale ptyId from the tab and fresh-spawn — else the next remount reads the same dead id and loops here. - try { - session.clearPaneMode2031State() - session.clearHiddenOutputRestoreState() - const outputCallbacks = session.captureTransportOutputCallbacks(session.reportError, null) - session.transport.attach({ - existingPtyId: attachPtyId, - cols: session.cols, - rows: session.rows, - callbacks: outputCallbacks.callbacks - }) - const attachedPtyId = session.transport.getPtyId() ?? attachPtyId - session.bindActivePanePty(attachedPtyId, { - updateTabPtyId: 'if-missing', - sampleVisibleForegroundAgent: true - }) - if (attachPtyId === eagerLivePtyId || isRemoteRuntimePtyId(attachedPtyId)) { - session.registerPaneSerializerFor(attachedPtyId) - } - } catch (err) { - session.reportError(err instanceof Error ? err.message : String(err)) - session.deps.clearTabPtyId(session.deps.tabId, attachPtyId) - session.startFreshSpawn() + // Why: surface synchronous attach failures via session.reportError so the pane shows a diagnostic instead of a blank surface. + // On throw, clear the stale ptyId from the tab and fresh-spawn — else the next remount reads the same dead id and loops here. + try { + session.clearPaneMode2031State() + session.clearHiddenOutputRestoreState() + const outputCallbacks = session.captureTransportOutputCallbacks(session.reportError, null) + session.transport.attach({ + existingPtyId: attachPtyId, + cols: session.cols, + rows: session.rows, + callbacks: outputCallbacks.callbacks + }) + const attachedPtyId = session.transport.getPtyId() ?? attachPtyId + session.bindActivePanePty(attachedPtyId, { + updateTabPtyId: 'if-missing', + sampleVisibleForegroundAgent: true + }) + if (attachPtyId === eagerLivePtyId || isRemoteRuntimePtyId(attachedPtyId)) { + session.registerPaneSerializerFor(attachedPtyId) } + } catch (err) { + session.reportError(err instanceof Error ? err.message : String(err)) + session.deps.clearTabPtyId(session.deps.tabId, attachPtyId) + session.startFreshSpawn() } } else { session.allowInitialIdleCacheSeed = false @@ -187,7 +175,7 @@ export function runDeferredSessionReattachChoice(session: ConnectPanePtySession) `Pending PTY spawn for tab ${session.deps.tabId} resolved without a PTY id, retrying fresh spawn` ) } - if (sleptRemoteColdRestoreStartup || hasSleepingAgentSession) { + if (sleptRemoteColdRestoreStartup || hasResumableSleepingAgentSession) { session.startFreshColdRestoreAgentResume(sleptRemoteColdRestoreStartup ?? undefined) } else { session.startFreshSpawn() @@ -218,7 +206,7 @@ export function runDeferredSessionReattachChoice(session: ConnectPanePtySession) }) } else { recordPtyConnectDiagnostic(`pane=${session.pane.id} -> FRESH SPAWN`) - if (sleptRemoteColdRestoreStartup || hasSleepingAgentSession) { + if (sleptRemoteColdRestoreStartup || hasResumableSleepingAgentSession) { session.startFreshColdRestoreAgentResume(sleptRemoteColdRestoreStartup ?? undefined) } else { session.startFreshSpawn() diff --git a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-connect.ts b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-connect.ts index 66cfc4f526b7..4f40820ee571 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-connect.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/deferred-session-reattach-connect.ts @@ -2,7 +2,10 @@ import { warnTerminalLifecycleAnomaly } from '../terminal-lifecycle-diagnostics' import { isSshSessionGoneError, recordPtyConnectDiagnostic } from './pty-connect-limits' import { isRemoteRuntimePtyId } from './paired-parked-terminal-restore' import { toProcessExitStartup } from './process-exit-startup' -import { recoverUnverifiableDirectSshReattach } from './direct-ssh-reattach-recovery' +import { + mayRetireBindingAfterFailedReattach, + recoverUnverifiableReattach +} from './unverifiable-reattach-recovery' import type { ConnectPanePtySession } from './connect-pane-pty-session' const PANE_OWNER_UNVERIFIED_ERROR = 'terminal_pane_owner_unverified' @@ -173,8 +176,8 @@ export function startDeferredSessionReattach( return } session.reportError(message) - if (session.connectionId) { - recoverUnverifiableDirectSshReattach(session, deferredReattachSessionId) + if (!mayRetireBindingAfterFailedReattach(session)) { + recoverUnverifiableReattach(session, deferredReattachSessionId) return } session.clearExitedPanePtyLayoutBinding(deferredReattachSessionId) diff --git a/src/renderer/src/components/terminal-pane/pty-connection/fresh-spawn-eligibility.ts b/src/renderer/src/components/terminal-pane/pty-connection/fresh-spawn-eligibility.ts new file mode 100644 index 000000000000..92f40b5bd7e1 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/pty-connection/fresh-spawn-eligibility.ts @@ -0,0 +1,9 @@ +import { useAppStore } from '@/store' +import type { ConnectPanePtySession } from './connect-pane-pty-session' + +export function mayStartFreshPaneSession(session: ConnectPanePtySession): boolean { + return ( + !session.isLegacyWorkerAutomaticResumeBlocked() && + !useAppStore.getState().deleteStateByWorktreeId?.[session.deps.worktreeId]?.isDeleting + ) +} diff --git a/src/renderer/src/components/terminal-pane/pty-connection/fresh-spawn-start.ts b/src/renderer/src/components/terminal-pane/pty-connection/fresh-spawn-start.ts index f3dab2d54258..1d570bb6b104 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/fresh-spawn-start.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/fresh-spawn-start.ts @@ -1,3 +1,4 @@ +import { mayStartFreshPaneSession } from './fresh-spawn-eligibility' import { useAppStore } from '@/store' import { hasPtySerializer } from '../pty-buffer-serializer' import { writeTerminalOutput } from '@/lib/pane-manager/pane-terminal-output-scheduler' @@ -34,15 +35,7 @@ export function bindStartFreshSpawn(session: ConnectPanePtySession): void { } } } - if (session.isLegacyWorkerAutomaticResumeBlocked()) { - releaseDeferredCwdFence() - return Promise.resolve(null) - } - if (useAppStore.getState().deleteStateByWorktreeId?.[session.deps.worktreeId]?.isDeleting) { - // Why: the worktree is being deleted; its PTYs were just killed for the - // filesystem teardown. A fresh shell must not spawn into a directory the - // removal is about to delete (main fences it anyway), and the pane is - // about to unmount — so skip the doomed respawn instead of racing it. + if (!mayStartFreshPaneSession(session)) { releaseDeferredCwdFence() return Promise.resolve(null) } diff --git a/src/renderer/src/components/terminal-pane/pty-connection/reattach-result-handler.ts b/src/renderer/src/components/terminal-pane/pty-connection/reattach-result-handler.ts index 6450a71567c1..577ab3af1c6c 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/reattach-result-handler.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/reattach-result-handler.ts @@ -18,7 +18,10 @@ import type { ConnectPanePtySession } from './connect-pane-pty-session' import type { ReattachPayloadContext } from './reattach-payload-context' import { createReattachPayloadHandlers } from './apply-reattach-payload' import type { ReattachPayloadSession } from './reattach-payload-session' -import { recoverUnverifiableDirectSshReattach } from './direct-ssh-reattach-recovery' +import { + mayRetireBindingAfterFailedReattach, + recoverUnverifiableReattach +} from './unverifiable-reattach-recovery' type ReattachResultSession = ReattachPayloadSession & Pick< @@ -91,7 +94,7 @@ export function bindHandleReattachResult(sessionBag: ConnectPanePtySession): voi } if (connectResult?.exitedBeforeAttach) { - // Why: the transport already delivered the dead session's final frame + exit; treat as terminal state, not a failed reattach. + // Preserve the historical pane when its owner observed exit before attachment. return true } @@ -105,7 +108,7 @@ export function bindHandleReattachResult(sessionBag: ConnectPanePtySession): voi const ptyId = connectResult?.id ?? (typeof result === 'string' ? result : session.transport.getPtyId()) const hasExplicitPtyId = Boolean(connectResult?.id || typeof result === 'string') - if (!ptyId) { + if (!ptyId || connectResult?.reattachUnverifiable) { warnTerminalLifecycleAnomaly('restored PTY reattach returned no PTY id', { tabId: session.deps.tabId, worktreeId: session.deps.worktreeId, @@ -113,19 +116,16 @@ export function bindHandleReattachResult(sessionBag: ConnectPanePtySession): voi paneId: session.pane.id, ptyId: staleSessionId ?? null }) - if (session.connectionId) { - recoverUnverifiableDirectSshReattach(sessionBag, staleSessionId) + if (connectResult?.reattachUnverifiable || !mayRetireBindingAfterFailedReattach(sessionBag)) { + recoverUnverifiableReattach(sessionBag, retryPtyId) return false } - // Why: a stale restored session can fail reattach after mount; don't leave xterm alive without a backing PTY. if (staleSessionId) { session.clearExitedPanePtyLayoutBinding(staleSessionId) + session.deps.clearTabPtyId(session.deps.tabId, staleSessionId) } else { session.syncPanePtyLayoutBinding(null) } - if (staleSessionId) { - session.deps.clearTabPtyId(session.deps.tabId, staleSessionId) - } session.startFreshColdRestoreAgentResume(coldRestoreStartup, { forceBlankRestoredViewport: true }) diff --git a/src/renderer/src/components/terminal-pane/pty-connection/reattach-unverifiable-result.test.ts b/src/renderer/src/components/terminal-pane/pty-connection/reattach-unverifiable-result.test.ts new file mode 100644 index 000000000000..5252d1e7f89a --- /dev/null +++ b/src/renderer/src/components/terminal-pane/pty-connection/reattach-unverifiable-result.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it, vi } from 'vitest' +import { bindHandleReattachResult } from './reattach-result-handler' +import { startDeferredSessionReattach } from './deferred-session-reattach-connect' +import { requestTerminalPaneRecovery } from '../terminal-pane-recovery' + +vi.mock('../terminal-pane-recovery', () => ({ requestTerminalPaneRecovery: vi.fn() })) +vi.mock('@/runtime/sync-runtime-graph', () => ({ scheduleRuntimeGraphSync: vi.fn() })) +vi.mock('@/store', () => ({ useAppStore: { getState: () => ({}) } })) + +function createSession() { + const transport = { getPtyId: () => null, connect: vi.fn() } + return { + transport, + pane: { id: 1 }, + deps: { + tabId: 'tab-1', + worktreeId: 'wt-1', + paneTransportsRef: { current: new Map([[1, transport]]) }, + clearTabPtyId: vi.fn() + }, + terminalRecoveryInstance: { id: 1 }, + terminalRecoveryGeneration: 1, + transportStreamGeneration: 1, + authoritativeReattachGeneration: 0, + isLegacyWorkerAutomaticResumeBlocked: () => false, + rejectObsoleteDirectSshReattach: () => false, + clearExitedPanePtyLayoutBinding: vi.fn(), + syncPanePtyLayoutBinding: vi.fn(), + startFreshColdRestoreAgentResume: vi.fn(), + handleReattachResult: vi.fn() + } +} + +describe('unverifiable reattach on every host', () => { + it.each([undefined, { id: 'retained-pty', reattachUnverifiable: true }])( + 'preserves local ownership for %j', + async (result) => { + vi.clearAllMocks() + const session = createSession() + session.isLegacyWorkerAutomaticResumeBlocked = () => !result + bindHandleReattachResult(session as never) + expect(await session.handleReattachResult(result, 'retained-pty')).toBe(false) + expect(session.deps.clearTabPtyId).not.toHaveBeenCalled() + expect(session.clearExitedPanePtyLayoutBinding).not.toHaveBeenCalled() + expect(session.startFreshColdRestoreAgentResume).not.toHaveBeenCalled() + expect(requestTerminalPaneRecovery).toHaveBeenCalledWith( + expect.objectContaining({ ptyId: 'retained-pty', reason: 'reattach-unverifiable' }) + ) + } + ) + + it('preserves local ownership when connect throws', async () => { + vi.clearAllMocks() + const base = createSession() + base.transport.connect.mockRejectedValue(new Error('daemon unavailable')) + const session = { + ...base, + runtimeEnvironmentId: 'env-1', + prepaintParkedSshSnapshot: vi.fn(), + buildColdRestoreAgentResumeStartup: () => null, + captureTransportOutputCallbacks: () => ({ generation: 1, callbacks: {} }), + beginReattachLiveDataDeferral: vi.fn(), + finishReattachLiveDataDeferral: vi.fn(), + shouldDeclareHiddenAtSpawn: () => false, + reportError: vi.fn(), + armDirectSshPaneRetryTimeout: vi.fn() + } + startDeferredSessionReattach(session as never, 'retained-pty') + await session.armDirectSshPaneRetryTimeout.mock.calls[0][0] + expect(session.deps.clearTabPtyId).not.toHaveBeenCalled() + expect(session.clearExitedPanePtyLayoutBinding).not.toHaveBeenCalled() + expect(session.startFreshColdRestoreAgentResume).not.toHaveBeenCalled() + expect(requestTerminalPaneRecovery).toHaveBeenCalledWith( + expect.objectContaining({ ptyId: 'retained-pty', reason: 'reattach-unverifiable' }) + ) + }) +}) diff --git a/src/renderer/src/components/terminal-pane/pty-connection/retained-legacy-pty-attach.ts b/src/renderer/src/components/terminal-pane/pty-connection/retained-legacy-pty-attach.ts deleted file mode 100644 index db1d277dcc3a..000000000000 --- a/src/renderer/src/components/terminal-pane/pty-connection/retained-legacy-pty-attach.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { isRemoteRuntimePtyId } from './paired-parked-terminal-restore' - -import type { ConnectPanePtySession } from './connect-pane-pty-session' - -export function bindAttachRetainedLegacyPty(session: ConnectPanePtySession): void { - session.attachRetainedLegacyPty = (ptyId: string): boolean => { - try { - session.authoritativeReattachGeneration += 1 - session.clearPaneMode2031State() - session.clearHiddenOutputRestoreState() - const outputCallbacks = session.captureTransportOutputCallbacks(session.reportError, null) - session.transport.attach({ - existingPtyId: ptyId, - callbacks: outputCallbacks.callbacks - }) - const attachedPtyId = session.transport.getPtyId() ?? ptyId - session.bindActivePanePty(attachedPtyId, { - updateTabPtyId: 'if-missing', - sampleVisibleForegroundAgent: true - }) - if (isRemoteRuntimePtyId(attachedPtyId)) { - session.registerPaneSerializerFor(attachedPtyId) - } - return true - } catch (err) { - session.reportError(err instanceof Error ? err.message : String(err)) - return false - } - } -} diff --git a/src/renderer/src/components/terminal-pane/pty-connection/run-deferred-connect.ts b/src/renderer/src/components/terminal-pane/pty-connection/run-deferred-connect.ts index efc1ef59b7c9..120805f9d291 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/run-deferred-connect.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/run-deferred-connect.ts @@ -12,7 +12,6 @@ import { bindPrepaintParkedSshSnapshot } from './ssh-snapshot-prepaint' import { bindForegroundOutputRefresh } from './foreground-output-refresh' import { bindRegisterPaneSerializer } from './pane-serializer-register' import { bindHandleReattachResult } from './reattach-result-handler' -import { bindAttachRetainedLegacyPty } from './retained-legacy-pty-attach' import { runDeferredSessionAttach } from './deferred-session-attach' import { bindSerializeHiddenOutputSnapshot } from './hidden-output-snapshot-serialize' @@ -154,7 +153,6 @@ export function installRunDeferredConnect(session: ConnectPanePtySession): void bindPrepaintParkedSshSnapshot(session) bindHandleReattachResult(session) - bindAttachRetainedLegacyPty(session) runDeferredSessionAttach(session) } diff --git a/src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts b/src/renderer/src/components/terminal-pane/pty-connection/unverifiable-reattach-recovery.test.ts similarity index 50% rename from src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts rename to src/renderer/src/components/terminal-pane/pty-connection/unverifiable-reattach-recovery.test.ts index 03f3da2f1587..aafeb21aad5e 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/unverifiable-reattach-recovery.test.ts @@ -1,12 +1,20 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { requestTerminalPaneRecovery } from '../terminal-pane-recovery' -import { recoverUnverifiableDirectSshReattach } from './direct-ssh-reattach-recovery' +import { + mayRetireBindingAfterFailedReattach, + recoverUnverifiableReattach +} from './unverifiable-reattach-recovery' + +const state = vi.hoisted(() => ({ + deleteStateByWorktreeId: {} as Record +})) +vi.mock('@/store', () => ({ useAppStore: { getState: () => state } })) vi.mock('../terminal-pane-recovery', () => ({ requestTerminalPaneRecovery: vi.fn() })) -describe('recoverUnverifiableDirectSshReattach', () => { +describe('recoverUnverifiableReattach', () => { beforeEach(() => { vi.clearAllMocks() }) @@ -15,7 +23,7 @@ describe('recoverUnverifiableDirectSshReattach', () => { const attempt = { attemptId: 'attempt-1' } const settleDirectSshPaneRetryAttempt = vi.fn() - recoverUnverifiableDirectSshReattach( + recoverUnverifiableReattach( { directSshRetryAttempt: attempt, settleDirectSshPaneRetryAttempt } as never, 'ssh:target@@pty-1' ) @@ -25,7 +33,7 @@ describe('recoverUnverifiableDirectSshReattach', () => { }) it('remounts over the preserved PTY when no retry lease exists', () => { - recoverUnverifiableDirectSshReattach( + recoverUnverifiableReattach( { directSshRetryAttempt: undefined, deps: { tabId: 'tab-1' }, @@ -44,3 +52,23 @@ describe('recoverUnverifiableDirectSshReattach', () => { }) }) }) + +describe('replacement eligibility after failed reattach', () => { + it.each([ + ['ordinary local', false, false, undefined, undefined, true], + ['fenced local', true, false, undefined, undefined, false], + ['deleting workspace', false, true, undefined, undefined, false], + ['direct SSH', false, false, 'ssh-1', undefined, false], + ['paired host', false, false, undefined, 'env-1', false] + ] as const)('%s', (_name, blocked, deleting, connectionId, runtimeEnvironmentId, expected) => { + state.deleteStateByWorktreeId = { 'wt-1': { isDeleting: deleting } } + expect( + mayRetireBindingAfterFailedReattach({ + deps: { worktreeId: 'wt-1' }, + connectionId, + runtimeEnvironmentId, + isLegacyWorkerAutomaticResumeBlocked: () => blocked + } as never) + ).toBe(expected) + }) +}) diff --git a/src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.ts b/src/renderer/src/components/terminal-pane/pty-connection/unverifiable-reattach-recovery.ts similarity index 62% rename from src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.ts rename to src/renderer/src/components/terminal-pane/pty-connection/unverifiable-reattach-recovery.ts index f32896fc254b..6270dc1572ac 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/direct-ssh-reattach-recovery.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/unverifiable-reattach-recovery.ts @@ -1,7 +1,13 @@ +import { mayStartFreshPaneSession } from './fresh-spawn-eligibility' import { requestTerminalPaneRecovery } from '../terminal-pane-recovery' import type { ConnectPanePtySession } from './connect-pane-pty-session' -export function recoverUnverifiableDirectSshReattach( +// Retire only when this transition is allowed to replace the local session. +export function mayRetireBindingAfterFailedReattach(session: ConnectPanePtySession): boolean { + return !session.connectionId && !session.runtimeEnvironmentId && mayStartFreshPaneSession(session) +} + +export function recoverUnverifiableReattach( session: ConnectPanePtySession, ptyId: string | null | undefined ): void { diff --git a/src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts b/src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts index 2ff2c9062a27..5efda0e91b3b 100644 --- a/src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-transport-spawn-errors.test.ts @@ -191,7 +191,7 @@ describe('createIpcPtyTransport', () => { expect(onError).not.toHaveBeenCalled() // undefined, not a sessionExpired result: the reattach handler's no-pty-id branch routes an - // SSH pane to recoverUnverifiableDirectSshReattach (remount + reattach, no shell restart). + // SSH pane to recoverUnverifiableReattach (remount + reattach, no shell restart). expect(result).toBeUndefined() }) diff --git a/src/renderer/src/components/terminal-pane/pty-transport-types.ts b/src/renderer/src/components/terminal-pane/pty-transport-types.ts index 4d7eaf7358b9..450b805c50f7 100644 --- a/src/renderer/src/components/terminal-pane/pty-transport-types.ts +++ b/src/renderer/src/components/terminal-pane/pty-transport-types.ts @@ -75,9 +75,9 @@ export type PtyConnectResult = { id: string /** Host-owned PTY incarnation used to fence remote identity observations. */ incarnationId?: string - /** The requested session exited while it had no primary pane handler. Its - * buffered final data/exit were delivered, so callers must not fresh-spawn. */ + /** The owner observed exit before attachment; preserve the pane without fresh-spawning. */ exitedBeforeAttach?: boolean + reattachUnverifiable?: boolean /** The provider adopted an existing session rather than creating a fresh one. * Startup commands may be ignored; recovery still requires separate ownership evidence. */ isReattach?: boolean diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pr-base-refusal-compatibility.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pr-base-refusal-compatibility.test.ts new file mode 100644 index 000000000000..705ee7cbeac7 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pr-base-refusal-compatibility.test.ts @@ -0,0 +1,113 @@ +import { resolve } from 'node:path' +import { TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { loadPrBasePtyTransport } from './pr-base-pty-transport-test-loader' +import { beforeEach, it, expect, vi } from 'vitest' +import { + createRemoteRuntimeTransportMocks, + type MultiplexSubscriptionCallbacks +} from './remote-runtime-pty-transport-test-harness' +let callbacks: MultiplexSubscriptionCallbacks = null +let handle = 'terminal-1' +const { runtimeCall, runtimeSubscribe, resetRemoteRuntimeTransport } = + createRemoteRuntimeTransportMocks({ + getCallbacks: () => callbacks, + setCallbacks: (c) => { + callbacks = c + }, + getResolvedPaneHandle: () => handle, + setResolvedPaneHandle: (h) => { + handle = h + } + }) +beforeEach(resetRemoteRuntimeTransport) + +it.each([ + ['old', 'exitedBeforeAttach'], + ['old', 'reattachUnverifiable'], + ['current', 'exitedBeforeAttach'], + ['current', 'reattachUnverifiable'] +] as const)('%s client preserves host %s refusal without a fake spawn', async (reader, outcome) => { + const hostPath = resolve(__dirname, '../../../../main/runtime') + const { terminalAttachRefusal } = await import(`${hostPath}/terminal-attach-refusal`) + const { mapRuntimeError } = await import(`${hostPath}/rpc/errors`) + const { TERMINAL_LIFECYCLE_METHODS } = await import( + `${hostPath}/rpc/methods/terminal/terminal-lifecycle-methods` + ) + const refused = terminalAttachRefusal( + { id: 'old-worker-pty', [outcome]: true }, + { + handle: 'unpublished-refusal-handle', + tabId: 'tab-1', + paneKey: 'tab-1:5b5b5b5b-5b5b-4b5b-8b5b-5b5b5b5b5b5b', + worktreeId: 'wt-1' + } + )! + runtimeCall.mockImplementation( + async ({ method, params }: { method: string; params: unknown }) => { + if (method === 'terminal.resolvePane') { + return { ok: false, error: { code: 'terminal_not_found', message: 'terminal_not_found' } } + } + if (method === 'status.get') { + return { ok: true, result: { capabilities: ['terminal.fenced-create.v1'] } } + } + if (method === 'terminal.create') { + const definition = TERMINAL_LIFECYCLE_METHODS.find( + (m: { name: string }) => m.name === 'terminal.create' + )! + try { + const result = await definition.handler(params, { + clientKind: 'runtime', + clientCapabilities: reader === 'old' ? [] : [TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY], + runtime: { + dedupeTerminalCreate: async ( + _client: string, + wt: string, + _id: unknown, + _reconcile: boolean, + create: (worktree: string, handle: string) => Promise + ) => create(wt, 'unpublished-refusal-handle'), + createTerminal: async () => refused + } + } as never) + return { ok: true, result } + } catch (error) { + return mapRuntimeError('request', { runtimeId: 'host' }, error) + } + } + return { ok: true, result: {} } + } + ) + const { createRemoteRuntimePtyTransport } = + reader === 'old' + ? await loadPrBasePtyTransport() + : await import('./remote-runtime-pty-transport') + const onPtySpawn = vi.fn() + const onExit = vi.fn() + const onError = vi.fn() + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: '5b5b5b5b-5b5b-4b5b-8b5b-5b5b5b5b5b5b', + onPtySpawn + }) + try { + const result = await transport.connect({ + url: '', + sessionId: 'remote:env-1@@retained-handle', + callbacks: { onExit, onError } + }) + expect(runtimeCall).toHaveBeenCalledWith(expect.objectContaining({ method: 'terminal.create' })) + expect(onPtySpawn).not.toHaveBeenCalled() + expect(onExit).not.toHaveBeenCalled() + expect(onError).not.toHaveBeenCalled() + if (reader === 'old') { + expect(result).toBeUndefined() + expect(transport.getRecoveryState?.()).toMatchObject({ phase: 'disconnected' }) + } else { + expect(result).toEqual({ id: 'remote:env-1@@retained-handle', [outcome]: true }) + } + expect(runtimeSubscribe).not.toHaveBeenCalled() + } finally { + transport.destroy?.() + } +}) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-fenced-outcome.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-fenced-outcome.test.ts new file mode 100644 index 000000000000..40677d45affa --- /dev/null +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport-fenced-outcome.test.ts @@ -0,0 +1,186 @@ +import { TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + createRemoteRuntimeTransportMocks, + type MultiplexSubscriptionCallbacks +} from './remote-runtime-pty-transport-test-harness' + +let subscriptionCallbacks: MultiplexSubscriptionCallbacks = null +let resolvedPaneHandle = 'terminal-1' + +const { runtimeCall, runtimeSubscribe, subscriptionSendBinary, resetRemoteRuntimeTransport } = + createRemoteRuntimeTransportMocks({ + getCallbacks: () => subscriptionCallbacks, + setCallbacks: (callbacks) => { + subscriptionCallbacks = callbacks + }, + getResolvedPaneHandle: () => resolvedPaneHandle, + setResolvedPaneHandle: (handle) => { + resolvedPaneHandle = handle + } + }) + +describe('paired host attach evidence', () => { + beforeEach(resetRemoteRuntimeTransport) + + it.each(['exitedBeforeAttach', 'reattachUnverifiable'] as const)( + 'carries host %s without subscribing or publishing a spawn', + async (outcome) => { + runtimeCall.mockImplementation(async ({ method }: { method: string }) => { + if (method === 'status.get') { + return { ok: true, result: { capabilities: [TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY] } } + } + if (method === 'terminal.resolvePane') { + return { + ok: false, + error: { + code: 'terminal_not_found', + message: 'terminal_not_found' + } + } + } + return { + ok: true, + result: { terminal: { handle: 'retained-handle', [outcome]: true } } + } + }) + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const onPtySpawn = vi.fn() + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1', + onPtySpawn + }) + const result = await transport.connect({ + url: '', + sessionId: 'remote:env-1@@retained-handle', + callbacks: {} + }) + expect(result).toEqual({ + id: 'remote:env-1@@retained-handle', + [outcome]: true + }) + expect(runtimeCall).toHaveBeenCalledWith( + expect.objectContaining({ method: 'terminal.create' }) + ) + expect(runtimeSubscribe).not.toHaveBeenCalled() + expect(onPtySpawn).not.toHaveBeenCalled() + transport.destroy?.() + } + ) + + it.each(['method_not_found', 'connection_unavailable'])( + 'preserves a native binding when pane resolution is %s', + async (code) => { + runtimeCall.mockResolvedValue({ + ok: false, + error: { code, message: code } + }) + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + const result = await transport.connect({ + url: '', + sessionId: 'ssh:host@@pty-1', + callbacks: {} + }) + expect(result).toEqual({ + id: 'ssh:host@@pty-1', + reattachUnverifiable: true + }) + expect(runtimeCall).not.toHaveBeenCalledWith( + expect.objectContaining({ method: 'terminal.create' }) + ) + transport.destroy?.() + } + ) + + it('subscribes the encoded handle when an older host lacks pane resolution', async () => { + runtimeCall.mockResolvedValue({ + ok: false, + error: { code: 'method_not_found', message: 'method_not_found' } + }) + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + const result = await transport.connect({ + url: '', + sessionId: 'remote:env-1@@terminal-1', + callbacks: {} + }) + expect(result).toMatchObject({ id: 'remote:env-1@@terminal-1', isReattach: true }) + expect(runtimeCall).not.toHaveBeenCalledWith( + expect.objectContaining({ method: 'terminal.create' }) + ) + expect(runtimeSubscribe).toHaveBeenCalled() + transport.destroy?.() + }) + + it('adopts an encoded handle through host resolution and subscription', async () => { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + const result = await transport.connect({ + url: '', + sessionId: 'remote:env-1@@terminal-1', + callbacks: {} + }) + expect(result).toMatchObject({ + id: 'remote:env-1@@terminal-1', + isReattach: true + }) + expect(runtimeCall).not.toHaveBeenCalledWith( + expect.objectContaining({ method: 'terminal.create' }) + ) + expect(runtimeSubscribe).toHaveBeenCalled() + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + transport.destroy?.() + }) + it('an old host with resolvePane must not create a replacement for a retained pane', async () => { + runtimeCall.mockImplementation(async ({ method }: { method: string }) => { + if (method === 'terminal.resolvePane') { + return { ok: false, error: { code: 'terminal_not_found', message: 'terminal_not_found' } } + } + return { + ok: true, + result: { + terminal: { + handle: 'replacement', + ptyId: 'replacement-pty', + tabId: 'tab-1', + paneKey: 'tab-1:pane:1' + } + } + } + }) + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + const result = await transport.connect({ + url: '', + sessionId: 'remote:env-1@@retained-handle', + callbacks: {} + }) + expect(result).toEqual({ id: 'remote:env-1@@retained-handle', reattachUnverifiable: true }) + try { + expect(runtimeCall).not.toHaveBeenCalledWith( + expect.objectContaining({ method: 'terminal.create' }) + ) + } finally { + transport.destroy?.() + } + }) +}) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts index 679208b37640..648f5947c3c3 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts @@ -19,7 +19,10 @@ import type { RuntimeTerminalResolvePane, RuntimeTerminalSend } from '../../../../shared/runtime-types' -import { TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { + TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY, + TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY +} from '../../../../shared/protocol-version' import { agentResumeHostAuthorityCapability } from '../../runtime/agent-resume-host-authority-capability' import { isTerminalInputTooLargeWithDeferredMeasurement, @@ -1197,13 +1200,17 @@ export function createRemoteRuntimePtyTransport( return null } - async function resolvePersistedHostPane(): Promise { + async function resolvePersistedHostPane(): Promise< + | { kind: 'resolved'; terminal: RuntimeTerminalResolvePane } + | { kind: 'owner-absent' } + | { kind: 'unavailable'; error?: unknown } + > { if (!tabId || !leafId || !worktreeId) { - return null + return { kind: 'unavailable' } } const paneKey = `${tabId}:${leafId}` if (resolvePaneUnavailable) { - return null + return { kind: 'unavailable' } } let terminal: RuntimeTerminalResolvePane try { @@ -1216,12 +1223,12 @@ export function createRemoteRuntimePtyTransport( const message = runtimeTerminalErrorMessage(error) if (error instanceof RuntimeRpcCallError && error.code === 'method_not_found') { resolvePaneUnavailable = true - return null + return { kind: 'unavailable' } } - if (message.includes('terminal_not_found') || message.includes('method_not_found')) { - return null + if (message === 'terminal_not_found') { + return { kind: 'owner-absent' } } - throw error + return { kind: 'unavailable', error } } if ( terminal.tabId !== tabId || @@ -1248,7 +1255,7 @@ export function createRemoteRuntimePtyTransport( throw new Error('terminal_owner_mismatch') } } - return terminal + return { kind: 'resolved', terminal } } async function adoptResolvedHostPane( @@ -1798,7 +1805,11 @@ export function createRemoteRuntimePtyTransport( ) return } else if (tabId && leafId && worktreeId) { - const resolved = await resolvePersistedHostPane() + const resolution = await resolvePersistedHostPane() + if (resolution.kind === 'unavailable' && resolution.error) { + throw resolution.error + } + const resolved = resolution.kind === 'resolved' ? resolution.terminal : null if (destroyed || !connected || handle !== previousHandle) { return } @@ -2190,14 +2201,52 @@ export function createRemoteRuntimePtyTransport( ) } - if (options.sessionId && !getRemoteRuntimeTerminalHandle(options.sessionId)) { - // Why: a HUB session persists host-native PTY ids; resolve its pane handle without exposing that SSH identity as a client transport id. - const terminal = await resolvePersistedHostPane() - if (terminal) { - return await adoptResolvedHostPane(terminal, options) + if (options.sessionId) { + const resolution = await resolvePersistedHostPane() + if (resolution.kind === 'resolved') { + return await adoptResolvedHostPane(resolution.terminal, options) + } + if (resolution.kind === 'unavailable') { + const persistedHandle = getRemoteRuntimeTerminalHandle(options.sessionId) + if ( + persistedHandle && + resolvePaneUnavailable && + getRemoteRuntimePtyEnvironmentId(options.sessionId) === currentRuntimeEnvironmentId + ) { + return await adoptResolvedHostPane( + { + handle: persistedHandle, + tabId: tabId ?? '', + leafId: leafId ?? '', + ptyId: null, + worktreeId + }, + options + ) + } + connecting = false + if ( + resolution.error && + isRecoverableRemoteRuntimeConnectionError( + toRemoteRuntimeClientErrorLike(resolution.error) + ) + ) { + scheduleConnectRetryAfterRecoverableFailure() + } else { + emitRecoveryState() + } + return { id: options.sessionId, reattachUnverifiable: true } + } + const status = await callRuntime('status.get') + if (destroyed || lifecycleEpoch !== connectLifecycleEpoch) { + return + } + if (!status.capabilities?.includes(TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY)) { + connecting = false + emitRecoveryState() + return { id: options.sessionId, reattachUnverifiable: true } } } - const commandToSend = options.command ?? command const startupCommandDeliveryToSend = options.startupCommandDelivery ?? startupCommandDelivery @@ -2319,6 +2368,18 @@ export function createRemoteRuntimePtyTransport( return } const createdTerminal = created.terminal + if (createdTerminal.exitedBeforeAttach || createdTerminal.reattachUnverifiable) { + connecting = false + emitRecoveryState() + return { + id: + options.sessionId ?? + toRemoteRuntimePtyId(createdTerminal.handle, currentRuntimeEnvironmentId), + ...(createdTerminal.exitedBeforeAttach + ? { exitedBeforeAttach: true } + : { reattachUnverifiable: true }) + } + } adoptExecutionMetadata(createdTerminal) if (created.disposition !== undefined && tabId && createdTerminal.tabId) { recordWebAgentSessionHandoff({ @@ -2463,7 +2524,11 @@ export function createRemoteRuntimePtyTransport( ) return } - const resolved = await resolvePersistedHostPane() + const resolution = await resolvePersistedHostPane() + if (resolution.kind === 'unavailable' && resolution.error) { + throw resolution.error + } + const resolved = resolution.kind === 'resolved' ? resolution.terminal : null if (generation !== attachGeneration || destroyed) { return } @@ -2487,7 +2552,11 @@ export function createRemoteRuntimePtyTransport( return } if (!resolved) { - surfaceErrorMessage('Remote terminal was closed.') + surfaceErrorMessage( + resolution.kind === 'owner-absent' + ? 'Remote terminal was closed.' + : 'Remote terminal is temporarily unavailable.' + ) return } await adoptResolvedHostPane(resolved, options, false, generation) diff --git a/src/shared/protocol-version.ts b/src/shared/protocol-version.ts index e1cf7594034f..99878a1c3a40 100644 --- a/src/shared/protocol-version.ts +++ b/src/shared/protocol-version.ts @@ -116,6 +116,7 @@ export const WORKTREE_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY = 'worktree.create-idempotency.v1' as const export const CODEX_RESET_CREDIT_RUNTIME_CAPABILITY = 'accounts.codex-reset-credit.v1' as const export const ACCOUNT_IMPORT_RUNTIME_CAPABILITY = 'accounts.import-host-credentials.v1' as const +export const TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY = 'terminal.fenced-create.v1' as const // Why: older hosts cannot reconcile terminal.create's mutation after losing the reply, so clients may only retry unknown outcomes when advertised. export const TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY = 'terminal.create-idempotency.v2' as const @@ -250,6 +251,7 @@ export const RUNTIME_CAPABILITIES = [ TERMINAL_QUICK_COMMANDS_RUNTIME_CAPABILITY, WORKTREE_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY, TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY, + TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY, SESSION_TAB_CLOSE_INTENT_RUNTIME_CAPABILITY, SESSION_TABS_AUTHORITATIVE_INVENTORY_RUNTIME_CAPABILITY, AGENT_SESSION_BOUNDARY_RUNTIME_CAPABILITY, diff --git a/src/shared/pty-attach-absence-evidence.ts b/src/shared/pty-attach-absence-evidence.ts index 2b306ca40f0a..e23e5a1edda2 100644 --- a/src/shared/pty-attach-absence-evidence.ts +++ b/src/shared/pty-attach-absence-evidence.ts @@ -15,3 +15,20 @@ const PROVEN_EXITED_ATTACH_REFUSAL = /PTY ".+" not found \(process exited\)/i export function isProvenExitedPtyAttachRefusal(error: unknown): boolean { return PROVEN_EXITED_ATTACH_REFUSAL.test(error instanceof Error ? error.message : String(error)) } + +/** + * The wording every PTY session owner (daemon host, local provider) mints when the id it was asked + * to attach is not in its own session map. Unlike the relay marker above this answer is + * unambiguous: the process that owns the session table answered about itself, so absence is + * `exited`, never `unverifiable` (docs/reference/ssh-execution-boundary.md). + * + * It crosses two process boundaries — daemon socket and Electron IPC — which erase the error class, + * so the text is the contract. Producers build their message from this constant. + */ +export const SESSION_NOT_FOUND_MESSAGE_PREFIX = 'Session not found: ' + +export function isSessionNotFoundRefusal(error: unknown): boolean { + return (error instanceof Error ? error.message : String(error)).includes( + SESSION_NOT_FOUND_MESSAGE_PREFIX + ) +} diff --git a/src/shared/remote-runtime-client-capabilities.ts b/src/shared/remote-runtime-client-capabilities.ts index 3797f08ffb9b..234521db6402 100644 --- a/src/shared/remote-runtime-client-capabilities.ts +++ b/src/shared/remote-runtime-client-capabilities.ts @@ -1,4 +1,5 @@ import { + TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY, AGENT_SESSION_BOUNDARY_RUNTIME_CAPABILITY, AUTOMATION_OWNER_FENCING_RUNTIME_CAPABILITY, SESSION_TAB_CLOSE_INTENT_RUNTIME_CAPABILITY, @@ -18,6 +19,7 @@ export function remoteRuntimeClientCapabilities( new Set([ SESSION_TAB_CLOSE_INTENT_RUNTIME_CAPABILITY, SESSION_TABS_AUTHORITATIVE_INVENTORY_RUNTIME_CAPABILITY, + TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY, AGENT_SESSION_BOUNDARY_RUNTIME_CAPABILITY, SKILL_INSTALL_RESULT_V2_CAPABILITY, WORKTREE_GITHUB_PR_SUPPRESSION_RUNTIME_CAPABILITY, diff --git a/src/shared/runtime-terminal-contracts.ts b/src/shared/runtime-terminal-contracts.ts index db1c3751ba82..3a0923504e00 100644 --- a/src/shared/runtime-terminal-contracts.ts +++ b/src/shared/runtime-terminal-contracts.ts @@ -274,6 +274,8 @@ export type RuntimeTerminalCreateRequestPayload = export type RuntimeTerminalCreate = { handle: string + exitedBeforeAttach?: true + reattachUnverifiable?: true /** Host-owned PTY incarnation used to fence remote identity observations. */ incarnationId?: string | null tabId?: string diff --git a/src/shared/terminal-fenced-create-capability.test.ts b/src/shared/terminal-fenced-create-capability.test.ts new file mode 100644 index 000000000000..3b7d7c9752d8 --- /dev/null +++ b/src/shared/terminal-fenced-create-capability.test.ts @@ -0,0 +1,8 @@ +import { expect, it } from 'vitest' +import { RUNTIME_CAPABILITIES, TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY } from './protocol-version' +import { remoteRuntimeClientCapabilities } from './remote-runtime-client-capabilities' + +it('advertises fenced create outcomes in host status and every paired client transport', () => { + expect(RUNTIME_CAPABILITIES).toContain(TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY) + expect(remoteRuntimeClientCapabilities()).toContain(TERMINAL_FENCED_CREATE_RUNTIME_CAPABILITY) +}) diff --git a/src/shared/terminal-pane-resume-fence.ts b/src/shared/terminal-pane-resume-fence.ts new file mode 100644 index 000000000000..40b11ea7780d --- /dev/null +++ b/src/shared/terminal-pane-resume-fence.ts @@ -0,0 +1,13 @@ +import type { WorkspaceSessionState } from './workspace-session-state-types' + +export function isPaneAutomaticResumeBlocked( + session: WorkspaceSessionState, + paneKey: string, + worktreeId: string +): boolean { + const record = session.sleepingAgentSessionsByPaneKey?.[paneKey] + return ( + record?.worktreeId === worktreeId && + record.automaticResumeBlockedBy === 'legacy-orchestration-worker' + ) +} diff --git a/tests/e2e/completed-worker-retirement-resume.spec.ts b/tests/e2e/completed-worker-retirement-resume.spec.ts index 6935cbe1895f..73c06fb2661e 100644 --- a/tests/e2e/completed-worker-retirement-resume.spec.ts +++ b/tests/e2e/completed-worker-retirement-resume.spec.ts @@ -7,7 +7,6 @@ import { } from './helpers/terminal' import { FAKE_AGENT_WINDOWS_SHELL } from './helpers/fake-agent-command-override' import { - cleanupCompletedWorkerFixture, clearCompletedWorkerLedger, completedWorkerFakeCodexCommand, completedWorkerLaunchEnv, @@ -31,10 +30,6 @@ const test = base.extend({ test.describe.configure({ mode: 'serial' }) -test.afterAll(() => { - cleanupCompletedWorkerFixture() -}) - for (const closeMode of ['terminal-close-cli', 'worker-release'] as const) { test(`completed background worker ${closeMode} retires resume authority before first activation`, async ({ orcaPage, diff --git a/tests/e2e/helpers/completed-worker-retirement-fixture.ts b/tests/e2e/helpers/completed-worker-retirement-fixture.ts index 43de5782648e..37b327c60a3c 100644 --- a/tests/e2e/helpers/completed-worker-retirement-fixture.ts +++ b/tests/e2e/helpers/completed-worker-retirement-fixture.ts @@ -94,9 +94,14 @@ export function clearCompletedWorkerLedger(): void { rmSync(lifecycleLedgerPath, { force: true }) } -export function cleanupCompletedWorkerFixture(): void { +// Why the module owns this and no spec may: `fakeCliDir` holds the fake `codex` executable every +// spec that imports this fixture launches. Playwright loads this module once per worker, so with +// one worker a per-file `afterAll` teardown deletes the binary out from under every later file in +// the run — the second spec then launches a deleted path, writes no lifecycle events, and times out +// waiting for them. The temp dir's lifetime is the worker process's, so only the process may end it. +process.once('exit', () => { rmSync(fakeCliDir, { recursive: true, force: true }) -} +}) export function readCompletedWorkerLedger(): LifecycleEvent[] { if (!existsSync(lifecycleLedgerPath)) { diff --git a/tests/e2e/settled-worker-tab-survives-restart.spec.ts b/tests/e2e/settled-worker-tab-survives-restart.spec.ts new file mode 100644 index 000000000000..97c620db7384 --- /dev/null +++ b/tests/e2e/settled-worker-tab-survives-restart.spec.ts @@ -0,0 +1,500 @@ +import { existsSync, readFileSync } from 'node:fs' +import path from 'node:path' +import { DaemonClient } from '../../src/main/daemon/client' +import { getDaemonSocketPath, getDaemonTokenPath } from '../../src/main/daemon/daemon-spawner' +import { DEFAULT_LOCAL_ORCA_PROFILE_ID } from '../../src/shared/orca-profiles' +import type { ElectronApplication, Page } from '@stablyai/playwright-test' +import { test, expect } from './helpers/orca-app' +import { TEST_REPO_PATH_FILE } from './global-setup' +import { attachRepoAndOpenTerminal, createRestartSession } from './helpers/orca-restart' +import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store' +import { + waitForActivePaneHookDescriptor, + waitForActivePanePtyId, + waitForActiveTerminalManager +} from './helpers/terminal' +import { FAKE_AGENT_WINDOWS_SHELL } from './helpers/fake-agent-command-override' +import { + clearCompletedWorkerLedger, + completedWorkerFakeCodexCommand, + completedWorkerLaunchEnv, + listRuntimeTerminals, + readCompletedWorkerDispatchCapability, + readCompletedWorkerLedger, + readPersistedWorkerRecoveryRecord, + seedCurrentCodexTranscript +} from './helpers/completed-worker-retirement-fixture' +import { RuntimeClient } from '../../src/cli/runtime-client' +import type { RuntimeTerminalSummary } from '../../src/shared/runtime-types' +import { splitWorktreeIdForFilesystem } from '../../src/shared/worktree/id' + +const PROVIDER_SESSION_ID = '019feb51-2269-71c2-89c6-faa8dc65c8dd' + +test.describe.configure({ mode: 'serial' }) + +async function findSecondaryWorktree( + page: Page, + client: RuntimeClient, + coordinatorWorktreeId: string +): Promise { + let targetWorktreeId: string | null = null + await expect + .poll( + async () => { + const listed = await client.call<{ worktrees: { id: string }[] }>('worktree.list', {}) + // The restart fixture only waits for the primary; refetch until the seeded secondary lands. + const rendererWorktreeIds = await page.evaluate(async () => { + const store = window.__store + if (!store) { + return [] + } + await Promise.all( + store.getState().repos.map((repo) => store.getState().fetchWorktrees(repo.id)) + ) + return Object.values(store.getState().worktreesByRepo) + .flat() + .map((worktree) => worktree.id) + }) + targetWorktreeId = + listed.result.worktrees.find( + (worktree) => + worktree.id !== coordinatorWorktreeId && rendererWorktreeIds.includes(worktree.id) + )?.id ?? null + return targetWorktreeId + }, + { timeout: 60_000, message: 'runtime never registered the secondary worktree' } + ) + .not.toBeNull() + if (!targetWorktreeId) { + throw new Error('The seeded repository did not expose its secondary worktree') + } + return targetWorktreeId +} + +async function backgroundMountTab(page: Page, worktreeId: string, tabId: string): Promise { + await page.evaluate( + ({ tabId, worktreeId }) => { + window.dispatchEvent( + new CustomEvent('orca-background-mount-terminal-worktree', { + detail: { worktreeId, tabIds: [tabId] } + }) + ) + }, + { tabId, worktreeId } + ) + await expect + .poll(() => page.evaluate((tabId) => Boolean(window.__paneManagers?.get(tabId)), tabId)) + .toBe(true) +} + +// A worker that reported done while its terminal stays open is fenced from auto-resume, so after an +// app restart its pane re-attaches renderer-only. Revealing that pane used to tear the tab down on a +// fabricated "no PTY" while the daemon still ran the process (#16904 regression). +for (const daemonSessionGone of [false, true]) { + test(`a settled worker tab survives restart with daemon session ${daemonSessionGone ? 'exited' : 'live'}`, async (// oxlint-disable-next-line no-empty-pattern -- Playwright's second fixture arg is testInfo; the first must be an object destructure to opt out of the default fixture set. + {}, testInfo) => { + test.setTimeout(300_000) + const repoPath = readFileSync(TEST_REPO_PATH_FILE, 'utf-8').trim() + if (!repoPath || !existsSync(repoPath)) { + test.skip(true, 'Global setup did not produce a seeded test repo') + return + } + clearCompletedWorkerLedger() + + const session = createRestartSession(testInfo, completedWorkerLaunchEnv) + let firstApp: ElectronApplication | null = null + let secondApp: ElectronApplication | null = null + try { + const first = await session.launch() + firstApp = first.app + const coordinatorWorktreeId = await attachRepoAndOpenTerminal(first.page, repoPath) + await waitForSessionReady(first.page) + await waitForActiveWorktree(first.page) + await ensureTerminalVisible(first.page) + await waitForActiveTerminalManager(first.page) + await waitForActivePanePtyId(first.page) + await first.page.evaluate( + async ({ agentCommand, terminalWindowsShell }) => { + await window.__store?.getState().updateSettings({ + agentCmdOverrides: { codex: agentCommand }, + terminalWindowsShell, + disabledTuiAgents: [], + terminalHiddenViewParking: false + }) + }, + { + agentCommand: completedWorkerFakeCodexCommand, + terminalWindowsShell: FAKE_AGENT_WINDOWS_SHELL + } + ) + const isolatedHome = await firstApp.evaluate(({ app }) => app.getPath('home')) + const client = new RuntimeClient(session.userDataDir, 30_000, null, null) + const coordinatorPane = await waitForActivePaneHookDescriptor(first.page) + const coordinatorHandle = ( + await client.call<{ terminal: { handle: string } }>('terminal.resolvePane', { + paneKey: coordinatorPane.paneKey + }) + ).result.terminal.handle + const targetWorktreeId = await findSecondaryWorktree( + first.page, + client, + coordinatorWorktreeId + ) + const targetWorktreePath = splitWorktreeIdForFilesystem(targetWorktreeId)?.worktreePath + if (!targetWorktreePath) { + throw new Error('The secondary worktree did not expose a filesystem path') + } + + const run = await client.call<{ run: { id: string } }>('orchestration.runCreate', { + objective: 'Keep one settled worker tab across restart', + from: coordinatorHandle + }) + const task = await client.call<{ task: { id: string } }>('orchestration.taskCreate', { + spec: 'Report completion and stay open', + run: run.result.run.id, + callerTerminalHandle: coordinatorHandle + }) + const started = await client.call<{ + dispatchId: string + state: string + effects: { kind: string; role?: string; id?: string }[] + }>('orchestration.workerStart', { + task: task.result.task.id, + from: coordinatorHandle, + worktree: `id:${targetWorktreeId}`, + agent: 'codex', + timeoutMs: 30_000 + }) + expect(started.result.state).toBe('ready') + const workerHandle = started.result.effects.find( + (effect) => effect.kind === 'terminal' && effect.role === 'agent' + )?.id + if (!workerHandle) { + throw new Error('worker-start did not return its agent terminal') + } + let worker: RuntimeTerminalSummary | undefined + await expect + .poll( + async () => { + worker = (await listRuntimeTerminals(client)).find( + (terminal) => terminal.handle === workerHandle + ) + return worker?.ptyId ?? null + }, + { timeout: 30_000, message: 'background worker never published its PTY identity' } + ) + .not.toBeNull() + if (!worker?.ptyId) { + throw new Error('Background worker did not publish its PTY') + } + const workerPtyId = worker.ptyId + const workerTabId = worker.tabId + const workerPaneKey = `${worker.tabId}:${worker.leafId}` + await backgroundMountTab(first.page, targetWorktreeId, workerTabId) + let dispatchCapability: string | null = null + await expect + .poll(() => { + dispatchCapability = readCompletedWorkerDispatchCapability() + return dispatchCapability + }) + .not.toBeNull() + if (!dispatchCapability) { + throw new Error('Background worker did not receive its dispatch capability') + } + const transcriptPath = seedCurrentCodexTranscript( + isolatedHome, + PROVIDER_SESSION_ID, + targetWorktreePath + ) + await first.page.evaluate( + ({ + agentCommand, + paneKey, + providerSessionId, + tabId, + terminalHandle, + transcriptPath, + worktreeId + }) => { + const state = window.__store?.getState() + if (!state) { + throw new Error('Renderer store unavailable') + } + const metadata = { tabId, worktreeId, terminalHandle } + const recovery = { + providerSession: { key: 'session_id' as const, id: providerSessionId, transcriptPath }, + launchConfig: { + agentCommand, + agentArgs: '--dangerously-bypass-approvals-and-sandbox', + agentEnv: {} + } + } + for (const agentState of ['working', 'done'] as const) { + state.setAgentStatus( + paneKey, + { state: agentState, prompt: 'Report completion and stay open', agentType: 'codex' }, + 'Settled background worker', + undefined, + metadata, + recovery + ) + } + }, + { + agentCommand: completedWorkerFakeCodexCommand, + paneKey: workerPaneKey, + providerSessionId: PROVIDER_SESSION_ID, + tabId: workerTabId, + terminalHandle: workerHandle, + transcriptPath, + worktreeId: targetWorktreeId + } + ) + const completed = await client.call<{ message: { type: string } }>( + 'orchestration.send', + { + from: workerHandle, + subject: 'Completed', + body: 'The fixture completed and stays open for inspection.', + type: 'worker_done', + payload: JSON.stringify({ + taskId: task.result.task.id, + dispatchId: started.result.dispatchId, + outcome: 'succeeded' + }) + }, + { orchestrationCapability: dispatchCapability } + ) + expect(completed.result.message.type).toBe('worker_done') + // The settlement sweep stamps the resume fence on the renderer's record before the tab closes. + await expect + .poll( + () => + first.page.evaluate( + (paneKey) => + window.__store?.getState().sleepingAgentSessionsByPaneKey[paneKey] + ?.automaticResumeBlockedBy ?? null, + workerPaneKey + ), + { timeout: 30_000, message: 'settled worker pane was never fenced' } + ) + .toBe('legacy-orchestration-worker') + + await session.close(firstApp) + firstApp = null + expect(readPersistedWorkerRecoveryRecord(session.userDataDir, workerPaneKey)).toMatchObject({ + automaticResumeBlockedBy: 'legacy-orchestration-worker' + }) + expect(readCompletedWorkerLedger().filter((event) => event.event === 'normal-exit')).toEqual( + [] + ) + + const launchesBeforeRestart = readCompletedWorkerLedger().filter( + (event) => event.event === 'spawn' + ) + if (daemonSessionGone) { + const daemonDir = path.join(session.userDataDir, 'daemon') + const daemon = new DaemonClient({ + socketPath: getDaemonSocketPath(daemonDir), + tokenPath: getDaemonTokenPath(daemonDir) + }) + try { + await daemon.ensureConnected() + await daemon.request('kill', { sessionId: workerPtyId, immediate: true }) + await expect + .poll(async () => { + const result = await daemon.request<{ sessions: { sessionId: string }[] }>( + 'listSessions', + undefined + ) + return result.sessions.some((entry) => entry.sessionId === workerPtyId) + }) + .toBe(false) + } finally { + daemon.disconnect() + } + } + const second = await session.launch() + secondApp = second.app + await waitForSessionReady(second.page) + if (!daemonSessionGone) { + // The daemon kept the worker alive across the app restart; the new main process never attached it. + await expect + .poll( + async () => + (await listRuntimeTerminals(client)).find( + (terminal) => terminal.ptyId === workerPtyId + )?.connected ?? null, + { timeout: 60_000, message: 'restarted runtime never rediscovered the worker PTY' } + ) + .toBe(true) + } + expect( + await second.page.evaluate( + ({ tabId, worktreeId }) => + Boolean( + window.__store?.getState().tabsByWorktree[worktreeId]?.some((tab) => tab.id === tabId) + ), + { tabId: workerTabId, worktreeId: targetWorktreeId } + ) + ).toBe(true) + + if (daemonSessionGone) { + await secondApp.evaluate(({ ipcMain }, ptyId) => { + type Handler = (event: unknown, args: Record) => Promise + const handlers = (ipcMain as unknown as { _invokeHandlers: Map }) + ._invokeHandlers + const spawn = handlers.get('pty:spawn') + if (!spawn) { + throw new Error('PTY spawn handler unavailable') + } + const observed = globalThis as typeof globalThis & { + __settledWorkerAttachExited?: boolean + } + handlers.set('pty:spawn', async (event, args) => { + const result = await spawn(event, args) + if (result && typeof result === 'object' && 'id' in result && result.id === ptyId) { + observed.__settledWorkerAttachExited = + 'exitedBeforeAttach' in result && result.exitedBeforeAttach === true + } + return result + }) + }, workerPtyId) + } + // Hidden mount, then reveal: the reveal is what runs the missing-session reconciler. + await backgroundMountTab(second.page, targetWorktreeId, workerTabId) + if (daemonSessionGone) { + await expect + .poll( + () => + secondApp!.evaluate( + () => + (globalThis as typeof globalThis & { __settledWorkerAttachExited?: boolean }) + .__settledWorkerAttachExited + ), + { timeout: 20_000, message: 'main must report observed exit without retiring the pane' } + ) + .toBe(true) + } + // Poll, don't sample: main's cache learns the session when the pane's deferred reattach lands, + // and backgroundMountTab only waits for the pane manager to exist. A restarted main that never + // attaches stays false for the whole window, which is the regression this guards. + if (!daemonSessionGone) { + await expect + .configure({ soft: true }) + .poll(() => second.page.evaluate((ptyId) => window.api.pty.hasPty(ptyId), workerPtyId), { + timeout: 20_000, + message: 'liveness before reveal' + }) + .toBe(true) + } + await second.page.evaluate( + ({ tabId, worktreeId }) => { + const store = window.__store + if (!store) { + throw new Error('Renderer store unavailable') + } + type Transition = { + activeWorktreeId: string | null + tabPresent: boolean + leafPtyIds: string[] + activeTabId: string | null + } + const snapshot = (state: ReturnType): Transition => ({ + activeWorktreeId: state.activeWorktreeId ?? null, + tabPresent: Boolean(state.tabsByWorktree[worktreeId]?.some((tab) => tab.id === tabId)), + leafPtyIds: Object.values(state.terminalLayoutsByTabId[tabId]?.ptyIdsByLeafId ?? {}), + activeTabId: state.activeTabIdByWorktree[worktreeId] ?? null + }) + const transitions: Transition[] = [snapshot(store.getState())] + const e2eWindow = window as typeof window & { __orcaRevealTransitions?: Transition[] } + e2eWindow.__orcaRevealTransitions = transitions + store.subscribe((state) => { + const next = snapshot(state) + if (JSON.stringify(next) !== JSON.stringify(transitions.at(-1))) { + transitions.push(next) + } + }) + store.getState().setActiveWorktree(worktreeId) + }, + { tabId: workerTabId, worktreeId: targetWorktreeId } + ) + // Give the reconciler's async verdict time to land; the tab must never have left. + await second.page.waitForTimeout(3_000) + const transitions = await second.page.evaluate( + () => + ( + window as typeof window & { + __orcaRevealTransitions?: { + activeWorktreeId: string | null + tabPresent: boolean + leafPtyIds: string[] + }[] + } + ).__orcaRevealTransitions ?? [] + ) + // Pre-fix this read: leaf binding cleared -> tab removed -> worktree deselected -> tab re-added by graph sync. + expect( + transitions.filter((step) => !step.tabPresent || step.leafPtyIds.length === 0), + 'reveal must not tear the settled worker tab down' + ).toEqual([]) + expect(transitions.at(-1)?.activeWorktreeId).toBe(targetWorktreeId) + expect( + await second.page.evaluate( + (tabId) => Boolean(window.__paneManagers?.get(tabId)), + workerTabId + ) + ).toBe(true) + if (!daemonSessionGone) { + expect( + (await listRuntimeTerminals(client)).find((terminal) => terminal.ptyId === workerPtyId) + ?.connected + ).toBe(true) + expect( + readCompletedWorkerLedger().filter((event) => event.event === 'normal-exit') + ).toEqual([]) + } + expect(readCompletedWorkerLedger().filter((event) => event.event === 'spawn')).toEqual( + launchesBeforeRestart + ) + expect( + await secondApp.evaluate(({ BrowserWindow }) => + BrowserWindow.getAllWindows().map((window) => ({ + visible: window.isVisible(), + focused: window.isFocused() + })) + ) + ).toEqual([{ visible: false, focused: false }]) + await second.page.screenshot({ path: testInfo.outputPath('settled-worker-revealed.png') }) + await session.close(secondApp) + secondApp = null + const persisted = JSON.parse( + readFileSync( + path.join( + session.userDataDir, + 'profiles', + DEFAULT_LOCAL_ORCA_PROFILE_ID, + 'orca-data.json' + ), + 'utf8' + ) + ).workspaceSession + expect( + persisted.tabsByWorktree[targetWorktreeId].some( + (tab: { id: string }) => tab.id === workerTabId + ) + ).toBe(true) + expect(Object.values(persisted.terminalLayoutsByTabId[workerTabId].ptyIdsByLeafId)).toContain( + workerPtyId + ) + } finally { + if (secondApp) { + await session.close(secondApp) + } + if (firstApp) { + await session.close(firstApp) + } + await session.dispose() + } + }) +} diff --git a/tests/fixtures/terminal/README.md b/tests/fixtures/terminal/README.md new file mode 100644 index 000000000000..c0a45e31480e --- /dev/null +++ b/tests/fixtures/terminal/README.md @@ -0,0 +1,10 @@ +`pr-base-1d1b73c40850-pty-transport.txt` is the exact +`src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts` from PR +#19358's base commit `1d1b73c40850`, excluding only its leading max-lines lint +suppression comment. Keep this historical reader frozen. + +The compatibility test compiles this data fixture as CommonJS with esbuild and +loads its dependencies through Vitest's mocked module graph. It exercises the +historical create-success/failure behavior without shipping the old transport or +adding a source line-cap exemption. It is not a full historical client binary; +transitive dependencies and stream doubles come from the current checkout. diff --git a/tests/fixtures/terminal/pr-base-1d1b73c40850-pty-transport.txt b/tests/fixtures/terminal/pr-base-1d1b73c40850-pty-transport.txt new file mode 100644 index 000000000000..d7c3a5b0aba4 --- /dev/null +++ b/tests/fixtures/terminal/pr-base-1d1b73c40850-pty-transport.txt @@ -0,0 +1,2782 @@ +import type { RuntimeRpcResponse } from '../../../../shared/runtime-rpc-envelope' +import { + isRecoverableRemoteRuntimeConnectionError, + isRuntimeRpcQueueOverloadError, + toRemoteRuntimeClientErrorLike +} from '../../../../shared/remote-runtime-client-error-classification' +import type { + RuntimeCreateAgentSessionResult, + RuntimeEnsureAgentSessionResult +} from '../../../../shared/agent-session-host-authority' +import type { ExecutionHostId } from '../../../../shared/execution-host' +import type { TabActivationIntent } from '../../../../shared/tab-activation-intent' +import type { + RuntimeMobileSessionTerminalClientTab, + RuntimeMobileSessionTabsResult, + RuntimeStatus, + RuntimeTerminalCreate, + RuntimeTerminalResolvePane, + RuntimeTerminalSend +} from '../../../../shared/runtime-types' +import { TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' +import { agentResumeHostAuthorityCapability } from '../../runtime/agent-resume-host-authority-capability' +import { + isTerminalInputTooLargeWithDeferredMeasurement, + iterateTerminalInputChunks +} from '../../../../shared/terminal-input' +import type { + IpcPtyTransportOptions, + PtyConnectResult, + PtyTransport, + PtyTransportRecoveryState +} from './pty-transport-types' +import { createPtyOutputProcessor } from './pty-transport' +import { isSshSessionGoneError } from './pty-connection/pty-connect-limits' +import { RuntimeRpcCallError, unwrapRuntimeRpcResult } from '../../runtime/runtime-rpc-client' +import { + getRemoteRuntimePtyEnvironmentId, + getRemoteRuntimeTerminalHandle, + runtimeTerminalErrorMessage, + toRemoteRuntimePtyId +} from '../../runtime/runtime-terminal-stream' +import { + getRemoteRuntimeTerminalMultiplexer, + REMOTE_TERMINAL_SNAPSHOT_TOO_LARGE, + type RemoteRuntimeMultiplexedTerminal, + type RemoteRuntimeSnapshotOutcome +} from '../../runtime/remote-runtime-terminal-multiplexer' +import { + toRuntimeTerminalWorktreeSelector, + toRuntimeWorktreeSelector +} from '../../runtime/runtime-worktree-selector' +import { + createRemoteRuntimePtyTextBatcher, + createRemoteRuntimeViewportBatcher +} from './remote-runtime-pty-batching' +import { + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS, + RemoteRuntimePtyRecoveryState +} from './remote-runtime-pty-recovery-state' +import { createBrowserUuid } from '@/lib/browser-uuid' +import { + createAgentSessionCreateOperation, + withAgentSessionCreateOperationId +} from '@/runtime/agent-session-create-operation' +import { replaceFitOverridePtyId, setFitOverride } from '@/lib/pane-manager/mobile-fit-overrides' +import { replaceDriverPtyId, setDriverForPty } from '@/lib/pane-manager/mobile-driver-state' +import { isWebTerminalSurfaceTabId, toHostSessionTabId } from '@/runtime/web-terminal-surface-id' +import { listRemoteRuntimeSessionTabsDeduped } from '@/runtime/remote-runtime-session-tabs-inflight' +import { subscribeAcceptedWebSessionTerminalHandle } from '@/runtime/web-session-terminal-handle-events' +import { runRemoteAgentSessionLaunch } from '@/runtime/remote-agent-session-launch' +import { useAppStore } from '@/store' +import { recordWebAgentSessionHandoff } from '@/runtime/web-agent-session-handoff' +import { refreshWebRuntimeSessionTabsSnapshot } from '@/runtime/web-runtime-session' +import { + bufferPtyShutdownData, + bufferPtyShutdownReplayData, + drainRolledBackPtyShutdownData, + isPtyDataHandlerShutdownPending, + ptyDataHandlers, + ptyReplayHandlers, + ptyShutdownLifecycleHandlers +} from './pty-shutdown-data-suspension' +import { getRuntimeEnvironmentRevision } from '@/runtime/runtime-environment-revision' + +const REMOTE_TERMINAL_INPUT_FLUSH_MS = 8 +const REMOTE_TERMINAL_VIEWPORT_FLUSH_MS = 33 +const REMOTE_RUNTIME_MAX_PENDING_QUERY_REPLIES = 64 +const HOST_SESSION_ATTACH_POLL_MS = 150 +const HOST_SESSION_POLL_MAX_MS = 1_000 +const HOST_SESSION_ATTACH_TIMEOUT_MS = 15_000 +const HOST_SESSION_INVENTORY_MAX_WINDOWS_PER_RECOVERY = 2 +const HOST_SESSION_SAME_HANDLE_END_REUSE_LIMIT = 2 +// Why its own constant: this fences how long an end-then-reattach on the same handle still counts as +// one recovery, which is unrelated to how long auto-recovery keeps retrying. It read the recovery +// budget before that budget became a derived value, and must not drift with it. +const HOST_SESSION_SAME_HANDLE_END_REUSE_WINDOW_MS = 60_000 +const MAX_SURFACED_TERMINAL_ERRORS = 8 +const TERMINAL_CREATE_RETRY_DELAYS_MS = [250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000] as const + +type HostHandleReplacementPolicy = 'reuse' | 'prefer-replacement' | 'require-replacement' + +type HostSessionHandleWaitResult = { + handle: string | null | undefined + inventoryFailed: boolean +} + +function stricterReplacementPolicy( + left: HostHandleReplacementPolicy, + right: HostHandleReplacementPolicy +): HostHandleReplacementPolicy { + const rank: Record = { + reuse: 0, + 'prefer-replacement': 1, + 'require-replacement': 2 + } + return rank[left] >= rank[right] ? left : right +} + +type RemoteAgentSessionLaunchResult = + | RuntimeEnsureAgentSessionResult + | RuntimeCreateAgentSessionResult + | { terminal: RuntimeTerminalCreate; disposition?: undefined } + +function isRemoteTerminalStaleMessage(message: string): boolean { + return message.includes('terminal_handle_stale') +} + +function isRemoteTerminalGoneMessage(message: string): boolean { + return ( + message.includes('terminal_exited') || + message.includes('terminal_gone') || + message.includes('no_connected_pty') || + message.toLocaleLowerCase('en-US').includes('explicitly killed') + ) +} + +/** PTY transport for a renderer pane backed by a terminal on a remote Orca runtime, over runtime RPC plus the multiplexed stream. */ +export function createRemoteRuntimePtyTransport( + runtimeEnvironmentId: string, + opts: IpcPtyTransportOptions = {} +): PtyTransport { + const { + command, + startupCommandDelivery, + env, + envToDelete, + launchConfig, + resumeProviderSession, + launchToken, + launchAgent, + terminalColorQueryReplies, + agentPrompt, + agentPromptDelivery, + agentArgsOverride, + agentLaunchPreferences, + worktreeId, + executionHostId, + tabId, + leafId, + activate, + onPtyExit, + onPtySpawn, + onPtyRebind, + onTitleChange, + onBell, + onAgentBecameIdle, + onAgentBecameWorking, + onAgentExited, + onAgentStatus + } = opts + let connected = false + let attachmentReady = false + let destroyed = false + let terminalEnded = false + let connecting = false + const attachmentReadyWaiters = new Set<(ready: boolean) => void>() + // Why: transport methods overlap during remounts; only the latest pane lifecycle may install a returned PTY. + let lifecycleEpoch = 0 + let handle: string | null = null + let remotePtyId: string | null = null + let authoritativeExecutionHostId: ExecutionHostId | null = executionHostId ?? null + let authoritativeHostPlatform: NodeJS.Platform | null = null + let authoritativePtyIncarnationId: string | null = null + let currentRuntimeEnvironmentId = runtimeEnvironmentId + const runtimeEnvironmentPairingRevision = getRuntimeEnvironmentRevision(runtimeEnvironmentId) + let multiplexedStream: RemoteRuntimeMultiplexedTerminal | null = null + let multiplexedStreamHandle: string | null = null + let desiredOutputPaused = false + let desiredViewport: { cols: number; rows: number } | null = null + let storedCallbacks: Parameters[0]['callbacks'] = {} + const surfacedErrorMessages = new Set() + let resubscribeEpoch: number | null = null + let resubscribeRequestedHandle: string | null = null + let resubscribeRequestedReplacementPolicy: HostHandleReplacementPolicy = 'reuse' + let recoveryReplacementPolicy: HostHandleReplacementPolicy = 'reuse' + let recoveryReplacementPolicyHandle: string | null = null + let stopWaitingForPublishedHandle: (() => void) | null = null + let publishedHandleWaitEpoch: number | null = null + // Why: a spent auto-recovery window is the evidence that licenses reattaching the fenced handle; explicit retries must not erase it. + let autoRecoveryWindowSpent = false + let settleHostSessionAttachRetry: ((retry: boolean) => void) | null = null + let resubscribeInventoryEpoch: number | null = null + let resubscribeInventoryWindows = 0 + let sameHandleEndReuseHandle: string | null = null + let sameHandleEndReuseCount = 0 + let sameHandleEndReuseAttachedAt: number | null = null + let attachGeneration = 0 + let subscriptionGeneration = 0 + + function setAttachmentReady(ready: boolean): void { + attachmentReady = ready + if (!ready) { + return + } + for (const resolve of attachmentReadyWaiters) { + resolve(true) + } + attachmentReadyWaiters.clear() + } + + function setAttachmentUnavailable(): void { + attachmentReady = false + for (const resolve of attachmentReadyWaiters) { + resolve(false) + } + attachmentReadyWaiters.clear() + } + + function waitForAttachmentReady(): Promise { + if (attachmentReady) { + return Promise.resolve(true) + } + if (destroyed || terminalEnded || !connected || !handle) { + return Promise.resolve(false) + } + return new Promise((resolve) => { + const timer = setTimeout(() => { + attachmentReadyWaiters.delete(settle) + resolve(false) + }, HOST_SESSION_ATTACH_TIMEOUT_MS) + const settle = (ready: boolean): void => { + clearTimeout(timer) + resolve(ready) + } + attachmentReadyWaiters.add(settle) + }) + } + + const recovery = new RemoteRuntimePtyRecoveryState(() => { + if (recovery.currentPhase === 'disposed') { + clearPublishedHandleWait() + } + if (recovery.currentPhase === 'disconnected') { + // Why: only the wall-clock deadline is evidence the window was spent; a UI latch from a fatal + // resubscribe must not license reattaching a fenced same handle (#12683). + autoRecoveryWindowSpent ||= recovery.autoRecoveryDeadlineExpired + // Why: cached pixels may remain, but no stream from the exhausted epoch may keep delivering or accepting terminal traffic. + subscriptionGeneration += 1 + closeMultiplexedStream() + } + if (recovery.currentPhase === 'idle') { + autoRecoveryWindowSpent = false + } + if ( + recovery.currentPhase === 'disconnected' || + recovery.currentPhase === 'disposed' || + recovery.currentPhase === 'idle' + ) { + settleHostSessionAttachRetry?.(false) + } + emitRecoveryState() + }) + let lastRecoveryStateKey = '' + let pendingViewportClaim = false + let pendingClaimInput: { text: string; queryReply: boolean }[] = [] + let pendingClaimQueryReplyCount = 0 + let terminalCreateRetryWait: { + timer: ReturnType + resolve: (continueRetrying: boolean) => void + } | null = null + // Why: after an unknown result, every later attempt must reconcile first so older runtimes cannot duplicate the PTY. + let terminalCreateNeedsReconciliation = false + // Why: once a structured outcome is ambiguous, only its stable host operation may be replayed. + let agentSessionRequiresHostAuthorityReplay = false + let terminalCreateUnknownOutcomeError: unknown = null + let lastConnectOptions: Parameters[0] | null = null + let lastAttachOptions: Parameters[0] | null = null + let resolvePaneUnavailable = false + let recoveringPaneHandle: string | null = null + const getRecoveryReplacementPolicy = (targetHandle: string): HostHandleReplacementPolicy => + recoveryReplacementPolicyHandle === targetHandle ? recoveryReplacementPolicy : 'reuse' + const resetRecoveryReplacementPolicy = (): void => { + recoveryReplacementPolicy = 'reuse' + recoveryReplacementPolicyHandle = null + } + const strengthenRecoveryReplacementPolicy = ( + targetHandle: string, + replacementPolicy: HostHandleReplacementPolicy + ): void => { + if (recoveryReplacementPolicyHandle !== targetHandle) { + recoveryReplacementPolicyHandle = targetHandle + recoveryReplacementPolicy = replacementPolicy + return + } + recoveryReplacementPolicy = stricterReplacementPolicy( + recoveryReplacementPolicy, + replacementPolicy + ) + } + const beginResubscribeInventoryWindow = (recoveryEpoch: number): number => { + if (resubscribeInventoryEpoch !== recoveryEpoch) { + resubscribeInventoryEpoch = recoveryEpoch + resubscribeInventoryWindows = 0 + } + resubscribeInventoryWindows += 1 + return resubscribeInventoryWindows + } + const resetSameHandleEndReuse = (): void => { + sameHandleEndReuseHandle = null + sameHandleEndReuseCount = 0 + sameHandleEndReuseAttachedAt = null + } + const recordSameHandleEndReuse = (targetHandle: string): void => { + if (sameHandleEndReuseHandle !== targetHandle) { + sameHandleEndReuseHandle = targetHandle + sameHandleEndReuseCount = 0 + } + sameHandleEndReuseCount += 1 + sameHandleEndReuseAttachedAt = Date.now() + } + const replacementPolicyAfterStreamEnd = (targetHandle: string): HostHandleReplacementPolicy => { + if ( + sameHandleEndReuseHandle !== targetHandle || + sameHandleEndReuseAttachedAt === null || + Date.now() - sameHandleEndReuseAttachedAt >= HOST_SESSION_SAME_HANDLE_END_REUSE_WINDOW_MS + ) { + resetSameHandleEndReuse() + return 'prefer-replacement' + } + return sameHandleEndReuseCount >= HOST_SESSION_SAME_HANDLE_END_REUSE_LIMIT + ? 'require-replacement' + : 'prefer-replacement' + } + const adoptExecutionMetadata = (terminal: { + executionHostId?: ExecutionHostId + hostPlatform?: NodeJS.Platform + incarnationId?: string | null + }): void => { + authoritativeExecutionHostId = terminal.executionHostId ?? authoritativeExecutionHostId + authoritativeHostPlatform = terminal.hostPlatform ?? authoritativeHostPlatform + authoritativePtyIncarnationId = terminal.incarnationId ?? null + } + const viewportClaimReadyWaiters = new Set<(ready: boolean) => void>() + const clearPendingViewportClaim = (): void => { + pendingViewportClaim = false + pendingClaimInput = [] + pendingClaimQueryReplyCount = 0 + for (const resolve of viewportClaimReadyWaiters) { + resolve(false) + } + viewportClaimReadyWaiters.clear() + } + const queuePendingClaimInput = (text: string, queryReply: boolean): void => { + if (queryReply && pendingClaimQueryReplyCount >= REMOTE_RUNTIME_MAX_PENDING_QUERY_REPLIES) { + const oldestReply = pendingClaimInput.findIndex((segment) => segment.queryReply) + if (oldestReply !== -1) { + pendingClaimInput.splice(oldestReply, 1) + pendingClaimQueryReplyCount -= 1 + const left = pendingClaimInput[oldestReply - 1] + const right = pendingClaimInput[oldestReply] + if (left && right && !left.queryReply && !right.queryReply) { + left.text += right.text + pendingClaimInput.splice(oldestReply, 1) + } + } + } + const tail = pendingClaimInput.at(-1) + if (!queryReply && tail && !tail.queryReply) { + tail.text += text + return + } + pendingClaimInput.push({ text, queryReply }) + if (queryReply) { + pendingClaimQueryReplyCount += 1 + } + } + // Why: clearing the claim flag without draining strands the queued bytes. + const flushPendingClaimInput = (stream: RemoteRuntimeMultiplexedTerminal): void => { + const queued = pendingClaimInput + pendingViewportClaim = false + pendingClaimInput = [] + pendingClaimQueryReplyCount = 0 + for (const segment of queued) { + stream.sendInput(segment.text) + } + for (const resolve of viewportClaimReadyWaiters) { + resolve(true) + } + viewportClaimReadyWaiters.clear() + } + // Why: tab/leaf ids are shared by paired viewers; the instance suffix keeps one viewer's refresh off peer records. + const clientId = `desktop:${tabId ?? 'tab'}:${leafId ?? 'leaf'}:${createBrowserUuid()}` + const terminalCreateMutationId = createBrowserUuid() + // Why: reconnect retries must replay one host operation instead of creating + // another fresh agent when the first response was lost. + const agentCreateOperation = createAgentSessionCreateOperation() + const outputProcessor = createPtyOutputProcessor({ + onTitleChange, + onBell, + onAgentBecameIdle, + onAgentBecameWorking, + onAgentExited, + onAgentStatus + }) + const shutdownDataHandler = ( + data: string, + meta?: Parameters[3] + ): void => { + outputProcessor.processData(data, storedCallbacks, undefined, meta) + } + const shutdownReplayHandler = (data: string): void => { + outputProcessor.processData(data, storedCallbacks, { + replayingBufferedData: true, + suppressAttentionEvents: true + }) + } + const shutdownLifecycle = { + pause: outputProcessor.pausePendingSideEffects, + rollback: outputProcessor.flushPendingSideEffects, + commit: outputProcessor.clearAccumulatedState + } + const registerShutdownHandlers = (ptyId: string): void => { + ptyDataHandlers.set(ptyId, shutdownDataHandler) + ptyReplayHandlers.set(ptyId, shutdownReplayHandler) + ptyShutdownLifecycleHandlers.set(ptyId, shutdownLifecycle) + if (!isPtyDataHandlerShutdownPending(ptyId)) { + drainRolledBackPtyShutdownData(ptyId) + } + } + const unregisterShutdownHandlers = (ptyId: string | null): void => { + if (!ptyId) { + return + } + if (ptyDataHandlers.get(ptyId) === shutdownDataHandler) { + ptyDataHandlers.delete(ptyId) + } + if (ptyReplayHandlers.get(ptyId) === shutdownReplayHandler) { + ptyReplayHandlers.delete(ptyId) + } + if (ptyShutdownLifecycleHandlers.get(ptyId) === shutdownLifecycle) { + ptyShutdownLifecycleHandlers.delete(ptyId) + } + } + + function getRecoveryState(): PtyTransportRecoveryState { + const phase = destroyed + ? 'disposed' + : terminalEnded + ? 'ended' + : recovery.currentPhase === 'recovering' + ? 'recovering' + : recovery.currentPhase === 'backoff' + ? 'backoff' + : recovery.currentPhase === 'disconnected' + ? 'disconnected' + : connecting + ? 'connecting' + : connected && attachmentReady + ? 'connected' + : 'offline' + return { + phase, + epoch: recovery.currentEpoch, + attempt: recovery.attemptCount + } + } + + function emitRecoveryState(force = false): void { + const state = getRecoveryState() + const key = `${state.phase}:${state.epoch}:${state.attempt}` + if (!force && key === lastRecoveryStateKey) { + return + } + lastRecoveryStateKey = key + storedCallbacks.onRecoveryStateChange?.(state) + } + + function surfaceErrorMessage(message: string): void { + if (surfacedErrorMessages.has(message)) { + return + } + while (surfacedErrorMessages.size >= MAX_SURFACED_TERMINAL_ERRORS) { + const oldest = surfacedErrorMessages.values().next().value + if (typeof oldest !== 'string') { + break + } + surfacedErrorMessages.delete(oldest) + } + surfacedErrorMessages.add(message) + storedCallbacks.onError?.(message) + } + + function markRecoveryHealthy(): void { + const recoveredErrors = [...surfacedErrorMessages] + surfacedErrorMessages.clear() + recovery.markHealthy() + for (const message of recoveredErrors) { + storedCallbacks.onErrorCleared?.(message) + } + } + + function hostSnapshotOwnsLaunch( + result: RemoteAgentSessionLaunchResult, + environmentId: string + ): boolean { + if (result.disposition !== undefined || result.terminal.isReattach === true) { + // Why: every structured launch is host-owned; provisional teardown must + // never close its canonical terminal while snapshot reconciliation catches up. + return true + } + const scopedPtyId = toRemoteRuntimePtyId(result.terminal.handle, environmentId) + return (useAppStore.getState().tabsByWorktree[worktreeId ?? ''] ?? []).some( + (tab) => + tab.ptyId === scopedPtyId || + (result.terminal.tabId !== undefined && + isWebTerminalSurfaceTabId(tab.id) && + toHostSessionTabId(tab.id) === result.terminal.tabId) + ) + } + + function findReadyHostSessionHandle( + snapshot: RuntimeMobileSessionTabsResult, + hostTabId: string + ): string | null { + const terminalTabs = getHostSessionTerminalSurfaces(snapshot, hostTabId, { + matchRequestedLeaf: false + }) + const selected = leafId + ? terminalTabs.find( + (tab) => tab.status === 'ready' && tab.parentTabId === hostTabId && tab.leafId === leafId + ) + : (terminalTabs.find( + (tab) => tab.status === 'ready' && tab.parentTabId === hostTabId && tab.isActive + ) ?? terminalTabs.find((tab) => tab.status === 'ready' && tab.parentTabId === hostTabId)) + if (selected?.status === 'ready') { + authoritativePtyIncarnationId = selected.incarnationId ?? null + return selected.terminal + } + return null + } + + function getHostSessionTerminalSurfaces( + snapshot: RuntimeMobileSessionTabsResult, + hostTabId: string, + options: { matchRequestedLeaf: boolean } + ): RuntimeMobileSessionTerminalClientTab[] { + return snapshot.tabs.filter( + (tab): tab is RuntimeMobileSessionTerminalClientTab => + tab.type === 'terminal' && + (tab.parentTabId === hostTabId || tab.id === hostTabId) && + (!options.matchRequestedLeaf || !leafId || tab.leafId === leafId) + ) + } + + function hasHostSessionTerminalSurface( + snapshot: RuntimeMobileSessionTabsResult, + hostTabId: string + ): boolean { + return ( + getHostSessionTerminalSurfaces(snapshot, hostTabId, { + matchRequestedLeaf: true + }).length > 0 + ) + } + + // Why: pending host surfaces materialize only through activation. + function activateHostSessionSurface( + hostTabId: string, + worktree: string, + intent: TabActivationIntent, + timeoutMs?: number + ): Promise { + return callRuntime( + 'session.tabs.activate', + { + worktree, + tabId: hostTabId, + ...(leafId ? { leafId } : {}), + notifyClients: false, + navigation: 'caller', + intent + }, + timeoutMs + ) + } + + function isMissingHostSessionSurfaceError(error: unknown): boolean { + const message = runtimeTerminalErrorMessage(error) + return message.includes('tab_not_found') || message.includes('terminal_not_found') + } + + async function waitForHostSessionHandle( + hostTabId: string, + isCurrent: () => boolean + ): Promise { + if (!worktreeId) { + return undefined + } + const worktree = toRuntimeWorktreeSelector(worktreeId) + let activated: RuntimeMobileSessionTabsResult + try { + // Why: this runs when the pane itself is opened/attached — the user's wake gesture. + activated = await activateHostSessionSurface(hostTabId, worktree, 'user') + } catch (error) { + if (isMissingHostSessionSurfaceError(error)) { + return null + } + throw error + } + const immediate = findReadyHostSessionHandle(activated, hostTabId) + if (immediate) { + return immediate + } + + const startedAt = Date.now() + let pollMs = HOST_SESSION_ATTACH_POLL_MS + let nextRequest: 'activate' | 'list' = 'list' + let activationOutcomeUnknown = false + while (isCurrent()) { + const remainingMs = HOST_SESSION_ATTACH_TIMEOUT_MS - (Date.now() - startedAt) + if (remainingMs <= 0) { + return undefined + } + // Why: host mirrors can publish before their PTY handle is ready, but a stuck pending surface must not poll forever. + await new Promise((resolve) => setTimeout(resolve, Math.min(pollMs, remainingMs))) + pollMs = Math.min(pollMs * 2, HOST_SESSION_POLL_MAX_MS) + // Why: an RPC left on its own 15s default would stretch this bounded wait to ~30s before the pane reports any terminal state. + const requestRemainingMs = HOST_SESSION_ATTACH_TIMEOUT_MS - (Date.now() - startedAt) + if (requestRemainingMs <= 0) { + return undefined + } + const request = nextRequest + let snapshot: RuntimeMobileSessionTabsResult + try { + snapshot = + request === 'list' + ? await listRemoteRuntimeSessionTabsDeduped({ + environmentId: currentRuntimeEnvironmentId, + worktreeId, + load: () => + callRuntime( + 'session.tabs.list', + { + worktree + }, + requestRemainingMs + ) + }) + : await activateHostSessionSurface(hostTabId, worktree, 'user', requestRemainingMs) + } catch (error) { + if (request === 'list') { + throw error + } + // Why: no re-activation failure is absence proof, whether the surface is missing or the host predates the method — but + // activation materializes host-side, so an outcome the client never saw must not be replayed into a duplicate PTY. + activationOutcomeUnknown = true + nextRequest = 'list' + continue + } + const handle = findReadyHostSessionHandle(snapshot, hostTabId) + if (handle) { + return handle + } + if (request === 'activate') { + // Why: an activation response can race host publication, so inventory — not this snapshot — decides what exists. + nextRequest = 'list' + continue + } + if (!hasHostSessionTerminalSurface(snapshot, hostTabId)) { + const siblingStillExists = + getHostSessionTerminalSurfaces(snapshot, hostTabId, { + matchRequestedLeaf: false + }).length > 0 + if (siblingStillExists) { + return false + } + // Why: a populated surface list missing only this leaf is positive absence, but a list carrying no + // surface at all for the tab is a client-side snapshot of a host that may still be republishing. + // Keep polling inside the bounded window and let it expire as unknown liveness, never as removal. + nextRequest = 'list' + continue + } + // Why: a host relaunch republishes the surface unmaterialized, and only activation can mint its PTY — list-only polling waits forever. + nextRequest = activationOutcomeUnknown ? 'list' : 'activate' + } + return undefined + } + + function waitForHostSessionAttachRetry(recoveryEpoch: number): Promise { + return new Promise((resolve) => { + let settled = false + const settle = (retry: boolean): void => { + if (settled) { + return + } + settled = true + if (settleHostSessionAttachRetry === settle) { + settleHostSessionAttachRetry = null + } + // Why: this wait is single-shot, so replaying it after the cutoff would strand the pane in 'recovering' with no RPC in flight. + recovery.discardPendingRetry(scheduledRetry) + resolve(retry) + } + const scheduledRetry = (): void => { + settle(true) + } + settleHostSessionAttachRetry?.(false) + settleHostSessionAttachRetry = settle + if (!recovery.schedule(recoveryEpoch, scheduledRetry)) { + settle(false) + } + }) + } + + async function waitForHostSessionHandleWithRecovery( + hostTabId: string, + isCurrent: () => boolean + ): Promise { + let recoveryEpoch = recovery.isActive ? recovery.currentEpoch : undefined + while (isCurrent()) { + try { + const hostHandle = await waitForHostSessionHandle(hostTabId, isCurrent) + if (!isCurrent()) { + return undefined + } + if (recoveryEpoch !== undefined) { + if (!recovery.isCurrent(recoveryEpoch)) { + return undefined + } + } + return hostHandle + } catch (error) { + if ( + !isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error)) || + !isCurrent() + ) { + throw error + } + if (recoveryEpoch !== undefined && !recovery.isCurrent(recoveryEpoch)) { + return undefined + } + recoveryEpoch ??= recovery.begin() + if (!(await waitForHostSessionAttachRetry(recoveryEpoch)) || !isCurrent()) { + return undefined + } + } + } + return undefined + } + + async function waitForResubscribeHostSessionHandle( + hostTabId: string, + previousHandle: string, + replacementPolicy: HostHandleReplacementPolicy, + recoveryEpoch: number + ): Promise { + if (!worktreeId) { + return { handle: null, inventoryFailed: false } + } + const worktree = toRuntimeWorktreeSelector(worktreeId) + const startedAt = Date.now() + let pollMs = HOST_SESSION_ATTACH_POLL_MS + // Why: list-only polling cannot recreate a host PTY lost across desktop generations. + let nextRequest: 'activate' | 'list' = 'activate' + let lastRequestError: unknown = null + let lastReadyHandle: string | null = null + const finishBoundedWait = (): HostSessionHandleWaitResult => { + const effectivePolicy = stricterReplacementPolicy( + replacementPolicy, + getRecoveryReplacementPolicy(previousHandle) + ) + if (effectivePolicy === 'prefer-replacement' && lastReadyHandle) { + return { handle: lastReadyHandle, inventoryFailed: false } + } + if (lastRequestError) { + console.warn( + '[remote-runtime-pty] host session recovery request failed during reconnect:', + runtimeTerminalErrorMessage(lastRequestError) + ) + } + // Why: a bounded wait without removal evidence is unknown liveness; keep the pane for a later snapshot to reattach. + return { handle: undefined, inventoryFailed: lastRequestError !== null } + } + while ( + !destroyed && + connected && + handle === previousHandle && + recovery.isCurrent(recoveryEpoch) + ) { + const requestRemainingMs = HOST_SESSION_ATTACH_TIMEOUT_MS - (Date.now() - startedAt) + if (requestRemainingMs <= 0) { + return finishBoundedWait() + } + const request = nextRequest + try { + const listed = + request === 'list' + ? await listRemoteRuntimeSessionTabsDeduped({ + environmentId: currentRuntimeEnvironmentId, + worktreeId, + load: () => + callRuntime( + 'session.tabs.list', + { + worktree + }, + requestRemainingMs + ) + }) + : // Why: reconnect recovery, not a user gesture — a pane the user slept + // must stay slept even though it publishes the same pending status. + await activateHostSessionSurface(hostTabId, worktree, 'automatic', requestRemainingMs) + lastRequestError = null + const nextHandle = findReadyHostSessionHandle(listed, hostTabId) + if (nextHandle) { + lastReadyHandle = nextHandle + } + const effectivePolicy = stricterReplacementPolicy( + replacementPolicy, + getRecoveryReplacementPolicy(previousHandle) + ) + if (nextHandle && (effectivePolicy === 'reuse' || nextHandle !== previousHandle)) { + return { handle: nextHandle, inventoryFailed: false } + } + if (request === 'list') { + if (!hasHostSessionTerminalSurface(listed, hostTabId)) { + return { handle: null, inventoryFailed: false } + } + if (!nextHandle) { + // Why: the surface is published but unmaterialized, and only activation can mint its PTY. + nextRequest = 'activate' + } + } else { + // Why: an activation response can race host publication, so inventory — not this snapshot — decides what exists. + nextRequest = 'list' + } + } catch (error) { + // Why: the inventory can race the reconnect that invalidated the handle; unknown liveness must not retire the pane. + lastRequestError = error + if (request === 'activate') { + // Why: no activation failure is absence proof, whether the surface is missing or the host predates the method. + nextRequest = 'list' + } + } + const remainingMs = HOST_SESSION_ATTACH_TIMEOUT_MS - (Date.now() - startedAt) + if (remainingMs <= 0) { + return finishBoundedWait() + } + // Why: a stale response can precede its replacement; bounded backoff avoids retrying the stale handle in a hot loop. + await new Promise((resolve) => setTimeout(resolve, Math.min(pollMs, remainingMs))) + pollMs = Math.min(pollMs * 2, HOST_SESSION_POLL_MAX_MS) + } + return { handle: undefined, inventoryFailed: false } + } + + // Why: the reconnect button and a parked external trigger revive a pane the same way — replay whichever entry point opened it. + function replayLastTransportEntryPoint(): boolean { + if (destroyed || terminalEnded || connected) { + return false + } + if (lastAttachOptions) { + transport.attach(lastAttachOptions) + return true + } + if (lastConnectOptions) { + void transport.connect(lastConnectOptions) + return true + } + return false + } + + // Why: a recoverable connect failure is unverifiable contact loss, not a dead terminal, so retry + // whichever path can still reach the pane instead of latching with nothing armed (#12684). + function retryAfterRecoverableConnectFailure(nextEpoch: number): void { + if (destroyed || terminalEnded) { + return + } + if (connected && handle) { + scheduleResubscribeAfterTransportClose(getRecoveryReplacementPolicy(handle), nextEpoch) + return + } + replayLastTransportEntryPoint() + } + + // Why: schedule() both auto-retries inside the window and leaves the retry parked when the deadline + // latches, so online/resume and the Reconnect button always find something to fire. + function scheduleConnectRetryAfterRecoverableFailure(): void { + if (destroyed) { + return + } + // Why: an ambiguous create already owns a reconciliation-gated retry that only Reconnect may + // re-enter; auto-replaying here would just re-probe a runtime that cannot reconcile. + if (terminalCreateNeedsReconciliation || agentSessionRequiresHostAuthorityReplay) { + recovery.markDisconnected() + return + } + // Why: the last attempt's RPC budget expires at the same instant as the deadline, so a silent drop + // rejects after the latch. Beginning a new epoch there re-arms the whole window, so park instead. + if (recovery.currentPhase === 'disconnected') { + recovery.parkRetryAfterDeadline(retryAfterRecoverableConnectFailure) + return + } + const recoveryEpoch = recovery.isActive ? recovery.currentEpoch : recovery.begin() + if (!recovery.schedule(recoveryEpoch, retryAfterRecoverableConnectFailure)) { + recovery.markDisconnected() + } + } + + async function attachHostSessionMirror( + options: { cols?: number; rows?: number }, + notifySpawn = true, + expectedAttachGeneration?: number, + expectedLifecycleEpoch?: number + ): Promise { + if (!tabId || !isWebTerminalSurfaceTabId(tabId)) { + return undefined + } + const isCurrent = (): boolean => + !destroyed && + (expectedAttachGeneration === undefined || expectedAttachGeneration === attachGeneration) && + (expectedLifecycleEpoch === undefined || expectedLifecycleEpoch === lifecycleEpoch) + const hostTabId = toHostSessionTabId(tabId) + const hostHandle = await waitForHostSessionHandleWithRecovery(hostTabId, isCurrent) + if (!isCurrent()) { + return undefined + } + if (hostHandle === undefined) { + connecting = false + // Why: no handle and no removal evidence is unknown liveness, so own an epoch and keep an unarmed retry parked for the + // banner and online/resume to fire — a silent 'connecting' renders no banner and retryRecovery() refuses to run. + if (recovery.currentPhase !== 'disconnected') { + const recoveryEpoch = recovery.isActive ? recovery.currentEpoch : recovery.begin() + recovery.parkRetryForExternalTrigger(recoveryEpoch, () => { + replayLastTransportEntryPoint() + }) + } + emitRecoveryState() + return undefined + } + if (!hostHandle) { + connecting = false + emitRecoveryState() + surfaceErrorMessage('Remote terminal was closed.') + return undefined + } + + if (leafId && worktreeId && !resolvePaneUnavailable) { + try { + const resolved = await callRuntime<{ terminal: RuntimeTerminalResolvePane }>( + 'terminal.resolvePane', + { paneKey: `${hostTabId}:${leafId}`, worktreeId } + ) + const terminal = resolved.terminal + if ( + terminal.handle === hostHandle && + terminal.tabId === hostTabId && + terminal.leafId === leafId && + (!terminal.worktreeId || terminal.worktreeId === worktreeId) + ) { + adoptExecutionMetadata(terminal) + } + } catch (error) { + if (error instanceof RuntimeRpcCallError && error.code === 'method_not_found') { + resolvePaneUnavailable = true + } + } + } + + if (!isCurrent() || recovery.currentPhase === 'disconnected') { + return undefined + } + handle = hostHandle + remotePtyId = toRemoteRuntimePtyId(hostHandle, currentRuntimeEnvironmentId) + registerShutdownHandlers(remotePtyId) + connected = true + desiredViewport = { + cols: options.cols ?? 80, + rows: options.rows ?? 24 + } + if (notifySpawn) { + onPtySpawn?.(remotePtyId) + } + + try { + await subscribeToHandle() + } catch (error) { + if (!recoverAfterSubscribeFailure(error, hostHandle, remotePtyId)) { + throw error + } + } + if (!connected || !remotePtyId || !isCurrent()) { + return undefined + } + + return { + id: remotePtyId, + replay: '', + isReattach: true, + ...(authoritativePtyIncarnationId ? { incarnationId: authoritativePtyIncarnationId } : {}) + } satisfies PtyConnectResult + } + + async function callRuntimeForEnvironment( + environmentId: string, + method: string, + params?: unknown, + timeoutMs = 15_000 + ): Promise { + const response = await window.api.runtimeEnvironments.call({ + selector: environmentId, + method, + params, + timeoutMs, + expectedEnvironmentPairingRevision: runtimeEnvironmentPairingRevision + }) + return unwrapRuntimeRpcResult(response as RuntimeRpcResponse) + } + + async function callRuntime( + method: string, + params?: unknown, + timeoutMs = 15_000 + ): Promise { + return callRuntimeForEnvironment(currentRuntimeEnvironmentId, method, params, timeoutMs) + } + + function cancelTerminalCreateRetryWait(): void { + const waiting = terminalCreateRetryWait + terminalCreateRetryWait = null + if (waiting) { + clearTimeout(waiting.timer) + waiting.resolve(false) + } + } + + function waitForTerminalCreateRetry(delayMs: number): Promise { + if (destroyed) { + return Promise.resolve(false) + } + return new Promise((resolve) => { + const timer = setTimeout(() => { + if (terminalCreateRetryWait?.timer === timer) { + terminalCreateRetryWait = null + } + resolve(!destroyed) + }, delayMs) + timer.unref?.() + terminalCreateRetryWait = { timer, resolve } + }) + } + + function terminalCreateRecoveryCutoffReached(): boolean { + return recovery.currentPhase === 'disconnected' + } + + async function createWithUnknownOutcomeRecovery( + kind: 'terminal' | 'agent-session', + invoke: ( + timeoutMs: number, + reconcileExisting: boolean + ) => Promise, + environmentId: string, + expectedLifecycleEpoch: number + ): Promise { + let retryAttempt = 0 + // Structured operations already carry their replay proof; ordinary terminal.create + // must prove v2 support before retrying an outcome the client cannot observe. + let idempotencySupported = kind === 'agent-session' + let reconcileExisting = + kind === 'agent-session' + ? agentSessionRequiresHostAuthorityReplay + : terminalCreateNeedsReconciliation + // Why the same budget: this loop calls recovery.begin(), so a shorter local deadline would abandon + // the create while the recovery state still reports 'recovering' with nothing in flight. + let recoveryDeadlineAt: number | null = recovery.isActive + ? Date.now() + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + : null + let lastError: unknown = + terminalCreateUnknownOutcomeError ?? new Error('Remote terminal creation was cancelled.') + while ( + !destroyed && + lifecycleEpoch === expectedLifecycleEpoch && + !terminalCreateRecoveryCutoffReached() + ) { + if (recoveryDeadlineAt !== null && recoveryDeadlineAt - Date.now() <= 0) { + break + } + while ( + reconcileExisting && + !idempotencySupported && + !destroyed && + lifecycleEpoch === expectedLifecycleEpoch && + !terminalCreateRecoveryCutoffReached() + ) { + let status: RuntimeStatus + try { + const statusRemainingMs = + recoveryDeadlineAt === null ? 5_000 : recoveryDeadlineAt - Date.now() + if (statusRemainingMs <= 0) { + break + } + status = await callRuntimeForEnvironment( + environmentId, + 'status.get', + undefined, + Math.min(5_000, statusRemainingMs) + ) + } catch (statusError) { + const statusClientError = toRemoteRuntimeClientErrorLike(statusError) + if (!isRecoverableRemoteRuntimeConnectionError(statusClientError)) { + throw statusError + } + const startsRecovery = recoveryDeadlineAt === null + recoveryDeadlineAt ??= Date.now() + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + if (startsRecovery && !recovery.isActive) { + recovery.begin() + } + const statusDelayMs = + TERMINAL_CREATE_RETRY_DELAYS_MS[ + Math.min(retryAttempt, TERMINAL_CREATE_RETRY_DELAYS_MS.length - 1) + ] + retryAttempt += 1 + const remainingMs = recoveryDeadlineAt - Date.now() + if ( + remainingMs <= 0 || + terminalCreateRecoveryCutoffReached() || + !(await waitForTerminalCreateRetry(Math.min(statusDelayMs, remainingMs))) + ) { + break + } + continue + } + if (!status.capabilities?.includes(TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY)) { + throw lastError + } + idempotencySupported = true + } + if ( + destroyed || + lifecycleEpoch !== expectedLifecycleEpoch || + (recoveryDeadlineAt !== null && recoveryDeadlineAt - Date.now() <= 0) + ) { + break + } + const createRemainingMs = recoveryDeadlineAt === null ? null : recoveryDeadlineAt - Date.now() + if (createRemainingMs !== null && createRemainingMs <= 0) { + break + } + try { + return await invoke(Math.min(15_000, createRemainingMs ?? 15_000), reconcileExisting) + } catch (error) { + lastError = error + const clientError = toRemoteRuntimeClientErrorLike(error) + if (!isRecoverableRemoteRuntimeConnectionError(clientError)) { + throw error + } + if (kind === 'agent-session') { + agentSessionRequiresHostAuthorityReplay = true + } else { + terminalCreateNeedsReconciliation = true + } + terminalCreateUnknownOutcomeError ??= error + reconcileExisting = true + const startsRecovery = recoveryDeadlineAt === null + recoveryDeadlineAt ??= Date.now() + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + if (startsRecovery && !recovery.isActive) { + recovery.begin() + } + if (destroyed || lifecycleEpoch !== expectedLifecycleEpoch) { + break + } + const remainingMs = recoveryDeadlineAt - Date.now() + if (remainingMs <= 0 || terminalCreateRecoveryCutoffReached()) { + break + } + const delayMs = + TERMINAL_CREATE_RETRY_DELAYS_MS[ + Math.min(retryAttempt, TERMINAL_CREATE_RETRY_DELAYS_MS.length - 1) + ] + retryAttempt += 1 + if (!(await waitForTerminalCreateRetry(Math.min(delayMs, remainingMs)))) { + break + } + } + } + return null + } + + async function resolvePersistedHostPane(): Promise { + if (!tabId || !leafId || !worktreeId) { + return null + } + const paneKey = `${tabId}:${leafId}` + if (resolvePaneUnavailable) { + return null + } + let terminal: RuntimeTerminalResolvePane + try { + const resolved = await callRuntime<{ terminal: RuntimeTerminalResolvePane }>( + 'terminal.resolvePane', + { paneKey, worktreeId } + ) + terminal = resolved.terminal + } catch (error) { + const message = runtimeTerminalErrorMessage(error) + if (error instanceof RuntimeRpcCallError && error.code === 'method_not_found') { + resolvePaneUnavailable = true + return null + } + if (message.includes('terminal_not_found') || message.includes('method_not_found')) { + return null + } + throw error + } + if ( + terminal.tabId !== tabId || + terminal.leafId !== leafId || + (terminal.worktreeId !== undefined && terminal.worktreeId !== worktreeId) + ) { + throw new Error('terminal_owner_mismatch') + } + if (terminal.worktreeId === undefined) { + const worktree = toRuntimeWorktreeSelector(worktreeId) + const listed = await listRemoteRuntimeSessionTabsDeduped({ + environmentId: currentRuntimeEnvironmentId, + worktreeId, + load: () => + callRuntime('session.tabs.list', { + worktree + }) + }) + const exactLegacyOwner = getHostSessionTerminalSurfaces(listed, tabId, { + matchRequestedLeaf: true + }).some((surface) => surface.status === 'ready' && surface.terminal === terminal.handle) + if (!exactLegacyOwner) { + // Why: legacy resolvePane responses lack worktree identity; only the scoped session snapshot can authorize adoption. + throw new Error('terminal_owner_mismatch') + } + } + return terminal + } + + async function adoptResolvedHostPane( + terminal: RuntimeTerminalResolvePane, + options: { cols?: number; rows?: number }, + notifySpawn = true, + expectedAttachGeneration?: number + ): Promise { + if ( + destroyed || + (expectedAttachGeneration !== undefined && expectedAttachGeneration !== attachGeneration) + ) { + return undefined + } + adoptExecutionMetadata(terminal) + const previousPtyId = remotePtyId + handle = terminal.handle + remotePtyId = toRemoteRuntimePtyId(handle, currentRuntimeEnvironmentId) + unregisterShutdownHandlers(previousPtyId) + registerShutdownHandlers(remotePtyId) + connected = true + desiredViewport = { + cols: options.cols ?? 80, + rows: options.rows ?? 24 + } + if (notifySpawn) { + onPtySpawn?.(remotePtyId) + } + emitRecoveryState() + try { + await subscribeToHandle() + } catch (error) { + if (!recoverAfterSubscribeFailure(error, handle, remotePtyId)) { + throw error + } + } + if ( + destroyed || + !connected || + !remotePtyId || + (expectedAttachGeneration !== undefined && expectedAttachGeneration !== attachGeneration) + ) { + return undefined + } + return { + id: remotePtyId, + replay: '', + isReattach: true, + ...(authoritativePtyIncarnationId ? { incarnationId: authoritativePtyIncarnationId } : {}) + } + } + + function recoverExpiredHostPane(): void { + const expiredHandle = handle + if (!expiredHandle || !tabId || !leafId || !worktreeId || recoveringPaneHandle) { + return + } + recoveringPaneHandle = expiredHandle + connected = false + clearPendingViewportClaim() + closeMultiplexedStream() + const hostTabId = isWebTerminalSurfaceTabId(tabId) ? toHostSessionTabId(tabId) : tabId + void callRuntime<{ terminal: RuntimeTerminalResolvePane }>('terminal.recoverPane', { + paneKey: `${hostTabId}:${leafId}`, + worktreeId, + expectedTerminal: expiredHandle + }) + .then(async ({ terminal }) => { + if (destroyed || handle !== expiredHandle) { + return + } + const previousIncarnationId = authoritativePtyIncarnationId + adoptExecutionMetadata(terminal) + const replacedPtyId = remotePtyId + handle = terminal.handle + remotePtyId = toRemoteRuntimePtyId(terminal.handle, currentRuntimeEnvironmentId) + unregisterShutdownHandlers(replacedPtyId) + registerShutdownHandlers(remotePtyId) + connected = true + if ( + replacedPtyId && + (replacedPtyId !== remotePtyId || previousIncarnationId !== authoritativePtyIncarnationId) + ) { + if (replacedPtyId !== remotePtyId) { + replaceFitOverridePtyId(replacedPtyId, remotePtyId) + replaceDriverPtyId(replacedPtyId, remotePtyId) + } + if (authoritativePtyIncarnationId) { + onPtyRebind?.(remotePtyId, replacedPtyId, authoritativePtyIncarnationId) + } else { + onPtyRebind?.(remotePtyId, replacedPtyId) + } + } + await subscribeToHandle() + }) + .catch((error) => { + if (!destroyed && handle === expiredHandle) { + surfaceErrorMessage(runtimeTerminalErrorMessage(error)) + } + }) + .finally(() => { + if (recoveringPaneHandle === expiredHandle) { + recoveringPaneHandle = null + } + }) + } + + async function closeRemoteTerminal( + handleOverride?: string, + environmentId = currentRuntimeEnvironmentId + ): Promise { + const targetHandle = handleOverride ?? handle + if (!targetHandle) { + return + } + try { + await callRuntimeForEnvironment(environmentId, 'terminal.close', { terminal: targetHandle }) + } catch { + // Best-effort parity with local disconnect/kill. + } + } + + function recoveryBlocksIo(): boolean { + return recovery.isActive || recovery.currentPhase === 'disconnected' + } + + async function sendInputAcceptedToRuntime(data: string): Promise { + const targetHandle = handle + if (!connected || !targetHandle || recoveryBlocksIo()) { + return false + } + if (!data) { + return true + } + await inputBatcher.drain() + if (!connected || handle !== targetHandle || recoveryBlocksIo()) { + return false + } + if (pendingViewportClaim && !getCurrentMultiplexedStream(targetHandle)) { + const ready = await new Promise((resolve) => { + viewportClaimReadyWaiters.add(resolve) + }) + if (!ready || !connected || handle !== targetHandle) { + return false + } + } + // Why: normal sendInput may be awaiting size validation; drain it before acknowledged writes so terminal bytes stay ordered. + const text = `${inputBatcher.takePending()}${data}` + try { + const tooLarge = isTerminalInputTooLargeWithDeferredMeasurement(text) + if (typeof tooLarge === 'boolean' ? tooLarge : await tooLarge) { + return false + } + } catch { + return false + } + try { + for (const chunk of iterateTerminalInputChunks(text)) { + if (!connected || handle !== targetHandle || recoveryBlocksIo()) { + return false + } + // Why: acknowledged sends order behind pending debounce text but must not collapse large paste back into one remote RPC. + const result = await callRuntime<{ send: RuntimeTerminalSend }>('terminal.send', { + terminal: targetHandle, + text: chunk, + client: { id: clientId, type: 'desktop' }, + ...(desiredViewport ? { viewport: desiredViewport, claimViewport: true as const } : {}) + }) + if (result.send.accepted !== true) { + return false + } + } + return true + } catch (error) { + // Why: stale-handle errors must retire the mirror (recoverable via next snapshot), not dead-end in a red xterm banner (#7718). + if (handle === targetHandle) { + handleRemoteTerminalError(error) + } + return false + } + } + + function notifyWriteUnavailable(): void { + if (!destroyed) { + storedCallbacks.onWriteUnavailable?.() + } + } + + const sendUnacknowledgedInput = (text: string, queryReply = false): boolean => { + const targetHandle = handle + const targetLifecycleEpoch = lifecycleEpoch + if (!connected || !targetHandle || recoveryBlocksIo()) { + return false + } + const stream = getCurrentMultiplexedStream(targetHandle) + if (stream?.sendInput(text)) { + return true + } + if (pendingViewportClaim) { + // Why: a claim during subscribe/reconnect has no stream record yet; hold its input so the stream emits claim+input in one order. + queuePendingClaimInput(text, queryReply) + return true + } + void callRuntime<{ send: RuntimeTerminalSend }>('terminal.send', { + terminal: targetHandle, + text, + client: { id: clientId, type: 'desktop' }, + ...(desiredViewport ? { viewport: desiredViewport, claimViewport: true as const } : {}) + }) + .then((result) => { + if ( + connected && + lifecycleEpoch === targetLifecycleEpoch && + handle === targetHandle && + result.send.accepted !== true + ) { + notifyWriteUnavailable() + } + }) + .catch((error) => { + if (lifecycleEpoch !== targetLifecycleEpoch || handle !== targetHandle) { + return + } + if (runtimeTerminalErrorMessage(error).includes('terminal_not_writable')) { + notifyWriteUnavailable() + } else { + handleRemoteTerminalError(error) + } + }) + return true + } + + const inputBatcher = createRemoteRuntimePtyTextBatcher( + REMOTE_TERMINAL_INPUT_FLUSH_MS, + sendUnacknowledgedInput + ) + + function sendViewportUpdate(cols: number, rows: number, claim = false): void { + const targetHandle = handle + if (!connected || !targetHandle || recoveryBlocksIo()) { + return + } + const stream = getCurrentMultiplexedStream(targetHandle) + if (claim ? stream?.claimViewport(cols, rows) : stream?.resize(cols, rows)) { + if (claim && stream) { + flushPendingClaimInput(stream) + } + return + } + if (claim) { + pendingViewportClaim = true + } + void callRuntime('terminal.updateViewport', { + terminal: targetHandle, + client: { id: clientId, type: 'desktop' }, + viewport: { cols, rows }, + ...(claim ? { claim: true } : {}) + }).catch(() => {}) + } + + const viewportBatcher = createRemoteRuntimeViewportBatcher( + REMOTE_TERMINAL_VIEWPORT_FLUSH_MS, + sendViewportUpdate + ) + + function rememberViewport(cols: number, rows: number): void { + desiredViewport = { cols, rows } + } + + function getCurrentMultiplexedStream( + targetHandle: string + ): RemoteRuntimeMultiplexedTerminal | null { + return multiplexedStreamHandle === targetHandle ? multiplexedStream : null + } + + function closeMultiplexedStream(): void { + multiplexedStream?.close() + multiplexedStream = null + multiplexedStreamHandle = null + setAttachmentReady(false) + } + + function clearPublishedHandleWait(): void { + stopWaitingForPublishedHandle?.() + stopWaitingForPublishedHandle = null + publishedHandleWaitEpoch = null + } + + function isCurrentRemoteTerminal(targetHandle: string, targetPtyId: string | null): boolean { + return ( + !destroyed && + connected && + handle === targetHandle && + remotePtyId === targetPtyId && + targetPtyId !== null + ) + } + + function retireRemoteTerminalId(exitCode?: number): void { + recovery.cancel() + resetRecoveryReplacementPolicy() + resetSameHandleEndReuse() + connected = false + connecting = false + terminalEnded = true + clearPublishedHandleWait() + clearPendingViewportClaim() + const stalePtyId = remotePtyId + unregisterShutdownHandlers(stalePtyId) + handle = null + remotePtyId = null + closeMultiplexedStream() + setAttachmentUnavailable() + emitRecoveryState() + if (stalePtyId) { + if (exitCode === undefined) { + onPtyExit?.(stalePtyId) + } else { + onPtyExit?.(stalePtyId, exitCode) + } + } + } + + function rebindRemoteTerminalHandle( + nextHandle: string, + nextIncarnationId: string | null = null + ): void { + clearPublishedHandleWait() + const replacedPtyId = remotePtyId + unregisterShutdownHandlers(replacedPtyId) + handle = nextHandle + remotePtyId = toRemoteRuntimePtyId(nextHandle, currentRuntimeEnvironmentId) + authoritativePtyIncarnationId = nextIncarnationId + resetRecoveryReplacementPolicy() + resetSameHandleEndReuse() + registerShutdownHandlers(remotePtyId) + setAttachmentReady(false) + // Why: host handle rotation preserves the pane generation; only the store identity changes, not spawn/exit semantics. + if (replacedPtyId) { + replaceFitOverridePtyId(replacedPtyId, remotePtyId) + replaceDriverPtyId(replacedPtyId, remotePtyId) + if (nextIncarnationId) { + onPtyRebind?.(remotePtyId, replacedPtyId, nextIncarnationId) + } else { + onPtyRebind?.(remotePtyId, replacedPtyId) + } + } + } + + function waitForPublishedHostSessionHandle(hostTabId: string, previousHandle: string): void { + if (!worktreeId) { + return + } + clearPublishedHandleWait() + stopWaitingForPublishedHandle = subscribeAcceptedWebSessionTerminalHandle( + { + environmentId: currentRuntimeEnvironmentId, + worktreeId, + hostTabId, + leafId + }, + (update) => { + if (destroyed || !connected || handle !== previousHandle) { + clearPublishedHandleWait() + return + } + if (!update.surfacePresent) { + // The host stopped publishing this surface, but that absence does + // not prove the remote process exited (the runtime may be restarting). + retireRemoteTerminalId(-1) + return + } + if (!update.terminalHandle) { + return + } + if (update.terminalHandle === previousHandle) { + // Why: once the auto-recovery window is spent, a host still publishing this surface is evidence the fenced handle outlived the stale error. + if (!autoRecoveryWindowSpent || getCurrentMultiplexedStream(previousHandle)) { + return + } + // Why: one reattach per spent window, so a handle that really is dead is not retried on every host snapshot. + autoRecoveryWindowSpent = false + const reattachEpoch = recovery.begin() + clearPublishedHandleWait() + const reusedPtyId = remotePtyId + void subscribeToHandle(reattachEpoch, true).catch((error) => { + if (!recoverAfterSubscribeFailure(error, previousHandle, reusedPtyId)) { + handleRemoteTerminalError(error) + } + }) + return + } + if (recovery.currentPhase === 'disconnected') { + // Why: without a live epoch a failed resubscribe is swallowed as already-latched, leaving a pane with no handle and no way back. + recovery.begin() + } + rebindRemoteTerminalHandle(update.terminalHandle) + const reboundHandle = handle + const reboundPtyId = remotePtyId + void subscribeToHandle().catch((error) => { + if (reboundHandle && !recoverAfterSubscribeFailure(error, reboundHandle, reboundPtyId)) { + handleRemoteTerminalError(error) + } + }) + } + ) + } + + function handleRemoteTerminalError(error: unknown): void { + const message = runtimeTerminalErrorMessage(error) + if (message === REMOTE_TERMINAL_SNAPSHOT_TOO_LARGE) { + // Why: an oversized initial snapshot is skipped but live output keeps flowing — informational, not fatal. + return + } + if (isRemoteTerminalStaleMessage(message)) { + if (tabId && leafId && worktreeId) { + // Why: reconnect can re-mint a pane handle while its host coordinates live; keep xterm state mounted while re-resolving. + closeMultiplexedStream() + scheduleResubscribeAfterTransportClose('require-replacement') + } else { + // A stale handle without a replacement is an attachment loss, not + // evidence that the process behind it died. + retireRemoteTerminalId(-1) + } + return + } + if (isRemoteTerminalGoneMessage(message)) { + // Why: an explicit terminal-gone response is lifecycle evidence, unlike a replaceable stale handle seen during reconnect. + retireRemoteTerminalId() + return + } + if (isSshSessionGoneError(message)) { + // Why: only the HUB may replace its expired SSH pane; a paired viewer must never fall back to + // client-local SSH. The identity-mismatch suffix is excluded because it means the opposite — + // the relay found a LIVE PTY under that id owned by another pane — and this is the one + // transport that actually calls terminal.recoverPane, so a bare substring test here spawned + // a second agent onto one transcript. + recoverExpiredHostPane() + return + } + const clientError = toRemoteRuntimeClientErrorLike(error) + if (isRuntimeRpcQueueOverloadError(clientError)) { + scheduleCapacityPressureRetry() + return + } + if (isRecoverableRemoteRuntimeConnectionError(clientError)) { + // Why: a partition is attachment state, not a terminal failure; keep the red error surface for actionable fatal errors. + scheduleResubscribeAfterTransportClose() + return + } + connecting = false + emitRecoveryState() + surfaceErrorMessage(message) + } + + function recoverAfterSubscribeFailure( + error: unknown, + targetHandle: string, + targetPtyId: string | null + ): boolean { + if (!isCurrentRemoteTerminal(targetHandle, targetPtyId)) { + return true + } + if (multiplexedStreamHandle !== targetHandle) { + closeMultiplexedStream() + } + clearPendingViewportClaim() + if (!isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error))) { + return false + } + if (recovery.currentPhase === 'disconnected') { + return true + } + scheduleResubscribeAfterTransportClose() + return true + } + + // Why: after a transport drop the host may have re-minted this handle; re-derive from the snapshot so we don't mirror/type into whatever PTY now sits behind the stale one (#7718). + async function resubscribeAfterTransportClose( + previousHandle: string, + replacementPolicy: HostHandleReplacementPolicy, + recoveryEpoch: number + ): Promise { + if (tabId && isWebTerminalSurfaceTabId(tabId)) { + const hostTabId = toHostSessionTabId(tabId) + const inventoryWindow = beginResubscribeInventoryWindow(recoveryEpoch) + const waitResult = await waitForResubscribeHostSessionHandle( + hostTabId, + previousHandle, + replacementPolicy, + recoveryEpoch + ) + const nextHandle = waitResult.handle + if ( + destroyed || + !connected || + handle !== previousHandle || + !recovery.isCurrent(recoveryEpoch) + ) { + return + } + if (nextHandle === undefined) { + const effectivePolicy = stricterReplacementPolicy( + replacementPolicy, + getRecoveryReplacementPolicy(previousHandle) + ) + if ( + effectivePolicy !== 'require-replacement' && + waitResult.inventoryFailed && + inventoryWindow < HOST_SESSION_INVENTORY_MAX_WINDOWS_PER_RECOVERY + ) { + throw Object.assign(new Error('Remote runtime session inventory polling failed.'), { + code: 'remote_runtime_unavailable' + }) + } + // Why: liveness is unknown, so auto-retry stops here; keep an unarmed retry parked for online/resume/reconnect to fire. + recovery.parkRetryForExternalTrigger(recoveryEpoch, (nextEpoch) => { + scheduleResubscribeAfterTransportClose( + handle ? getRecoveryReplacementPolicy(handle) : 'reuse', + nextEpoch + ) + }) + return + } + if (!nextHandle) { + // Why: host no longer publishes this surface; retire quietly and let the next session-tabs snapshot drive respawn/removal. + // Inventory absence is not process-liveness evidence; keep the tab + // recoverable until a later authoritative snapshot settles it. + retireRemoteTerminalId(-1) + return + } + const effectivePolicy = stricterReplacementPolicy( + replacementPolicy, + getRecoveryReplacementPolicy(previousHandle) + ) + // Why: a stale error can strengthen policy after inventory returns but before this continuation runs. + if (effectivePolicy === 'require-replacement' && nextHandle === previousHandle) { + return + } + if (nextHandle !== previousHandle) { + rebindRemoteTerminalHandle(nextHandle) + } + clearPublishedHandleWait() + await subscribeToHandle( + recoveryEpoch, + nextHandle === previousHandle && effectivePolicy === 'prefer-replacement' + ) + return + } else if (tabId && leafId && worktreeId) { + const resolved = await resolvePersistedHostPane() + if (destroyed || !connected || handle !== previousHandle) { + return + } + const effectivePolicy = stricterReplacementPolicy( + replacementPolicy, + getRecoveryReplacementPolicy(previousHandle) + ) + if ( + !resolved || + (effectivePolicy === 'require-replacement' && resolved.handle === previousHandle) + ) { + // A failed pane resolution leaves process liveness unknown. + retireRemoteTerminalId(-1) + return + } + if (resolved.handle !== previousHandle) { + adoptExecutionMetadata(resolved) + rebindRemoteTerminalHandle(resolved.handle, resolved.incarnationId ?? null) + } + clearPublishedHandleWait() + await subscribeToHandle( + recoveryEpoch, + resolved.handle === previousHandle && effectivePolicy === 'prefer-replacement' + ) + return + } + clearPublishedHandleWait() + await subscribeToHandle(recoveryEpoch) + } + + function scheduleResubscribeAfterTransportClose( + replacementPolicy: HostHandleReplacementPolicy = 'reuse', + requestedRecoveryEpoch?: number + ): void { + if (destroyed || !connected || !handle) { + return + } + const recoveryWasActive = recovery.isActive + const recoveryEpoch = requestedRecoveryEpoch ?? recovery.begin() + if (!recovery.isCurrent(recoveryEpoch)) { + return + } + if (!recoveryWasActive) { + // Why: bytes queued before a partition have unknown delivery; never replay them on a replacement stream. + inputBatcher.clear() + viewportBatcher.clear() + clearPendingViewportClaim() + } + strengthenRecoveryReplacementPolicy(handle, replacementPolicy) + if ( + replacementPolicy === 'require-replacement' && + stopWaitingForPublishedHandle && + // Why: only the epoch that handed recovery to accepted snapshots is blocked; a newer epoch is a fresh attempt, not a repeated stale send. + publishedHandleWaitEpoch === recoveryEpoch + ) { + return + } + if (resubscribeEpoch === recoveryEpoch) { + // Why: concurrent stale errors belong to their own handle; don't carry an old handle's replacement requirement onto its successor. + if (resubscribeRequestedHandle !== handle) { + resubscribeRequestedHandle = handle + resubscribeRequestedReplacementPolicy = replacementPolicy + } else { + resubscribeRequestedReplacementPolicy = stricterReplacementPolicy( + resubscribeRequestedReplacementPolicy, + replacementPolicy + ) + } + return + } + const resubscribeHandle = handle + clearPublishedHandleWait() + if (tabId && isWebTerminalSurfaceTabId(tabId)) { + // Why: subscribe before polling so a fresh host snapshot can't land in the gap between the inventory loop and its event-driven fallback. + waitForPublishedHostSessionHandle(toHostSessionTabId(tabId), resubscribeHandle) + publishedHandleWaitEpoch = recoveryEpoch + } + resubscribeEpoch = recoveryEpoch + resubscribeRequestedHandle = null + resubscribeRequestedReplacementPolicy = 'reuse' + let retryScheduled = false + void resubscribeAfterTransportClose(resubscribeHandle, replacementPolicy, recoveryEpoch) + .catch((error) => { + if (!destroyed && connected && handle && recovery.isCurrent(recoveryEpoch)) { + clearPendingViewportClaim() + const clientError = toRemoteRuntimeClientErrorLike(error) + if (isRecoverableRemoteRuntimeConnectionError(clientError)) { + retryScheduled = recovery.schedule(recoveryEpoch, (nextEpoch) => { + const currentReplacementPolicy = handle + ? getRecoveryReplacementPolicy(handle) + : 'reuse' + scheduleResubscribeAfterTransportClose(currentReplacementPolicy, nextEpoch) + }) + } else { + recovery.markDisconnected() + // Why: stale/gone/SSH-expired handling lives in handleRemoteTerminalError; its + // fallthrough surfaces the message, so routing here keeps those recoveries alive. + handleRemoteTerminalError(error) + } + } + }) + .finally(() => { + if (resubscribeEpoch !== recoveryEpoch) { + return + } + resubscribeEpoch = null + const pendingHandle = resubscribeRequestedHandle + const pendingReplacementPolicy = resubscribeRequestedReplacementPolicy + resubscribeRequestedHandle = null + resubscribeRequestedReplacementPolicy = 'reuse' + if ( + !retryScheduled && + recovery.isCurrent(recoveryEpoch) && + !stopWaitingForPublishedHandle && + pendingHandle && + pendingHandle === handle && + !getCurrentMultiplexedStream(pendingHandle) + ) { + scheduleResubscribeAfterTransportClose(pendingReplacementPolicy) + } + }) + } + + function scheduleCapacityPressureRetry(): void { + if (destroyed || !connected || !handle) { + return + } + const recoveryWasActive = recovery.isActive + const recoveryEpoch = recovery.begin() + if (!recoveryWasActive) { + inputBatcher.clear() + viewportBatcher.clear() + clearPendingViewportClaim() + } + recovery.schedule(recoveryEpoch, (nextEpoch) => { + scheduleResubscribeAfterTransportClose('reuse', nextEpoch) + }) + } + + async function subscribeToHandle( + expectedRecoveryEpoch?: number, + sameHandleEndRecovery = false + ): Promise { + if (!handle) { + return + } + const subscribedHandle = handle + const subscribedPtyId = remotePtyId + const generation = ++subscriptionGeneration + setAttachmentReady(false) + let transportClosed = false + let subscriptionAttached = false + let subscriptionSnapshotHadContent = false + // Why: viewport handed to subscribe; a resize during the round-trip falls back to the refresh-only one-shot RPC, replayed through the stream below once current. + const subscribedViewport = desiredViewport + const isCurrentSubscription = (): boolean => + !transportClosed && + generation === subscriptionGeneration && + (expectedRecoveryEpoch === undefined || recovery.ownsEpoch(expectedRecoveryEpoch)) && + isCurrentRemoteTerminal(subscribedHandle, subscribedPtyId) + const nextStream = await getRemoteRuntimeTerminalMultiplexer( + currentRuntimeEnvironmentId + ).subscribeTerminal({ + terminal: subscribedHandle, + client: { id: clientId, type: 'desktop' }, + viewport: subscribedViewport ?? undefined, + callbacks: { + onData: (data, meta) => { + if (isCurrentSubscription()) { + if (subscribedPtyId && bufferPtyShutdownData(subscribedPtyId, data, meta)) { + return + } + shutdownDataHandler(data, meta) + } + }, + onSnapshot: (data, meta) => { + // Why: an empty snapshot can still carry a pending mid-escape tail that must replay so the next live chunk completes it. + if ((data || meta?.pendingEscapeTailAnsi) && isCurrentSubscription()) { + subscriptionSnapshotHadContent = true + if (subscribedPtyId && bufferPtyShutdownReplayData(subscribedPtyId, data)) { + return + } + outputProcessor.processData(data, storedCallbacks, { + replayingBufferedData: true, + suppressAttentionEvents: true, + ...(meta?.pendingEscapeTailAnsi + ? { pendingEscapeTailAnsi: meta.pendingEscapeTailAnsi } + : {}), + // Why both or neither: the host's flags describe this image's own + // boundary, so an unsequenced snapshot proves nothing. + ...(meta?.kittyKeyboardFlags !== undefined && meta.seq !== undefined + ? { + kittyKeyboardFlags: meta.kittyKeyboardFlags, + snapshotSeq: meta.seq + } + : {}), + ...(meta?.terminalOwner && meta.seq !== undefined + ? { terminalOwner: meta.terminalOwner } + : {}), + ...(meta?.alternateScreen !== undefined && meta.seq !== undefined + ? { alternateScreen: meta.alternateScreen } + : {}), + // Why unconditional on seq: the grid describes the image itself, + // not a stream boundary, so it is valid for every snapshot the + // host dimensions. Absent/zero degrades to the pane's own grid. + ...(meta?.cols !== undefined && meta.rows !== undefined + ? { snapshotCols: meta.cols, snapshotRows: meta.rows } + : {}) + }) + } + }, + onOutputPauseCapability: () => { + if (isCurrentSubscription()) { + storedCallbacks.onOutputPauseChanged?.( + desiredOutputPaused, + nextStream.setOutputPaused(desiredOutputPaused) + ) + } + }, + onSubscribed: () => { + if (!isCurrentSubscription()) { + return + } + storedCallbacks.onOutputPauseChanged?.( + desiredOutputPaused, + nextStream.setOutputPaused(desiredOutputPaused) + ) + if (!subscriptionAttached && sameHandleEndRecovery) { + recordSameHandleEndReuse(subscribedHandle) + } + subscriptionAttached = true + setAttachmentReady(true) + connecting = false + resetRecoveryReplacementPolicy() + markRecoveryHealthy() + emitRecoveryState() + storedCallbacks.onConnect?.() + // Why: a recovery subscribe replays nothing when the host's push snapshot is + // empty (idle or exited pane), so ask for the retained buffer instead of + // waiting for bytes that an exited process will never send. + if (expectedRecoveryEpoch !== undefined && !subscriptionSnapshotHadContent) { + storedCallbacks.onStreamRecovered?.() + } + storedCallbacks.onStatus?.('shell') + }, + onEnd: (verdict) => { + if (!isCurrentSubscription()) { + return + } + outputProcessor.clearAccumulatedState() + if (verdict === 'unverifiable' || (tabId && isWebTerminalSurfaceTabId(tabId))) { + setAttachmentReady(false) + multiplexedStream = null + multiplexedStreamHandle = null + clearPendingViewportClaim() + // Why: a legacy bare end or waiter failure proves only attachment loss; bounded same-handle reuse preserves the tab without looping forever. + scheduleResubscribeAfterTransportClose( + replacementPolicyAfterStreamEnd(subscribedHandle) + ) + return + } + unregisterShutdownHandlers(subscribedPtyId) + connected = false + connecting = false + handle = null + remotePtyId = null + multiplexedStream = null + multiplexedStreamHandle = null + setAttachmentUnavailable() + terminalEnded = true + clearPendingViewportClaim() + emitRecoveryState() + storedCallbacks.onExit?.(0) + storedCallbacks.onDisconnect?.() + if (subscribedPtyId) { + onPtyExit?.(subscribedPtyId) + } + }, + onError: (message) => { + if (isCurrentSubscription()) { + handleRemoteTerminalError(message) + } + }, + onFitOverrideChanged: (event) => { + if (isCurrentSubscription() && subscribedPtyId) { + setFitOverride(subscribedPtyId, event.mode, event.cols, event.rows) + } + }, + onDriverChanged: (driver) => { + if (isCurrentSubscription() && subscribedPtyId) { + setDriverForPty(subscribedPtyId, driver) + } + }, + onWriteUnavailable: () => { + if (isCurrentSubscription()) { + notifyWriteUnavailable() + } + }, + onTransportClose: ({ recoverable, retryWithBackoff }) => { + transportClosed = true + if (generation !== subscriptionGeneration) { + return + } + if (!isCurrentSubscription()) { + // isCurrentSubscription excludes the just-closed stream by design. + if (!isCurrentRemoteTerminal(subscribedHandle, subscribedPtyId)) { + return + } + } + multiplexedStream = null + multiplexedStreamHandle = null + setAttachmentReady(false) + resetSameHandleEndReuse() + if (recoverable) { + if (retryWithBackoff) { + scheduleCapacityPressureRetry() + } else { + scheduleResubscribeAfterTransportClose() + } + } else { + connecting = false + recovery.cancel() + setAttachmentUnavailable() + emitRecoveryState() + } + } + } + }) + if ( + transportClosed || + generation !== subscriptionGeneration || + (expectedRecoveryEpoch !== undefined && !recovery.ownsEpoch(expectedRecoveryEpoch)) || + destroyed || + !connected || + handle !== subscribedHandle || + remotePtyId !== subscribedPtyId + ) { + nextStream.close() + return + } + closeMultiplexedStream() + multiplexedStream = nextStream + multiplexedStreamHandle = subscribedHandle + setAttachmentReady(subscriptionAttached) + if (subscriptionAttached) { + resetRecoveryReplacementPolicy() + markRecoveryHealthy() + } + // Why: a viewport change during the subscribe round-trip hit the no-op one-shot fallback; replay the latest viewport so the PTY isn't stuck at subscribe-time size. + if (pendingViewportClaim && desiredViewport) { + nextStream.claimViewport(desiredViewport.cols, desiredViewport.rows) + } else if ( + desiredViewport && + (desiredViewport.cols !== subscribedViewport?.cols || + desiredViewport.rows !== subscribedViewport?.rows) + ) { + nextStream.resize(desiredViewport.cols, desiredViewport.rows) + } + // Why: a live claim may already have cleared the flag, so drain on every install. + flushPendingClaimInput(nextStream) + } + + const transport: PtyTransport = { + async connect(options) { + cancelTerminalCreateRetryWait() + const connectLifecycleEpoch = ++lifecycleEpoch + const createEnvironmentId = currentRuntimeEnvironmentId + lastConnectOptions = options + lastAttachOptions = null + storedCallbacks = options.callbacks + resetRecoveryReplacementPolicy() + resetSameHandleEndReuse() + terminalEnded = false + connecting = true + emitRecoveryState(true) + if (destroyed || !worktreeId) { + return + } + + try { + if (isWebTerminalSurfaceTabId(tabId ?? '')) { + // Reattach callbacks must not publish the replacement as a fresh + // spawn before the reattach result commits tab/layout ownership. + return await attachHostSessionMirror( + options, + !options.sessionId, + undefined, + connectLifecycleEpoch + ) + } + + if (options.sessionId && !getRemoteRuntimeTerminalHandle(options.sessionId)) { + // Why: a HUB session persists host-native PTY ids; resolve its pane handle without exposing that SSH identity as a client transport id. + const terminal = await resolvePersistedHostPane() + if (terminal) { + return await adoptResolvedHostPane(terminal, options) + } + } + + const commandToSend = options.command ?? command + const startupCommandDeliveryToSend = + options.startupCommandDelivery ?? startupCommandDelivery + const envToSend = options.env ?? env + const envToDeleteToSend = options.envToDelete ?? envToDelete + const launchConfigToSend = options.launchConfig ?? launchConfig + const resumeProviderSessionToSend = options.resumeProviderSession ?? resumeProviderSession + const launchTokenToSend = options.launchToken ?? launchToken + const launchAgentToSend = options.launchAgent ?? launchAgent + const legacyCreateParams = { + worktree: toRuntimeTerminalWorktreeSelector(worktreeId), + clientMutationId: terminalCreateMutationId, + ...(commandToSend !== undefined ? { command: commandToSend } : {}), + ...(startupCommandDeliveryToSend !== undefined + ? { startupCommandDelivery: startupCommandDeliveryToSend } + : {}), + ...(envToSend !== undefined ? { env: envToSend } : {}), + ...(envToDeleteToSend !== undefined ? { envToDelete: envToDeleteToSend } : {}), + ...(launchConfigToSend !== undefined ? { launchConfig: launchConfigToSend } : {}), + ...(resumeProviderSessionToSend !== undefined + ? { resumeProviderSession: resumeProviderSessionToSend } + : {}), + ...(launchTokenToSend !== undefined ? { launchToken: launchTokenToSend } : {}), + ...(launchAgentToSend !== undefined ? { launchAgent: launchAgentToSend } : {}), + ...(terminalColorQueryReplies ? { terminalColorQueryReplies } : {}), + tabId, + leafId, + focus: false, + // Why: transport backs an already-mounted pane; activation is local state, not permission for remote UI reveal. + presentation: 'background' as const, + ...(activate === true ? { activate: true } : {}) + } + const legacyCreate = () => + createWithUnknownOutcomeRecovery( + 'terminal', + (timeoutMs, reconcileExisting) => + callRuntimeForEnvironment<{ terminal: RuntimeTerminalCreate }>( + createEnvironmentId, + 'terminal.create', + { + ...legacyCreateParams, + ...(reconcileExisting ? { reconcileExisting: true } : {}) + }, + timeoutMs + ), + createEnvironmentId, + connectLifecycleEpoch + ) + const hostAuthorityCreate = () => + createWithUnknownOutcomeRecovery( + 'agent-session', + (timeoutMs) => + resumeProviderSessionToSend + ? callRuntimeForEnvironment( + createEnvironmentId, + 'terminal.ensureAgentSession', + { + kind: 'explicit', + worktree: toRuntimeTerminalWorktreeSelector(worktreeId), + agent: launchAgentToSend!, + providerSession: resumeProviderSessionToSend, + ...(launchConfigToSend?.ompResumeFilePath + ? { ompResumeFilePath: launchConfigToSend.ompResumeFilePath } + : {}), + ...(agentArgsOverride !== undefined ? { agentArgs: agentArgsOverride } : {}), + ...(agentLaunchPreferences + ? { launchPreferences: agentLaunchPreferences } + : {}), + placement: { tabId, leafId }, + presentation: 'background' + }, + timeoutMs + ) + : callRuntimeForEnvironment( + createEnvironmentId, + 'terminal.createAgentSession', + withAgentSessionCreateOperationId( + { + worktree: toRuntimeTerminalWorktreeSelector(worktreeId), + agent: launchAgentToSend!, + ...(agentPrompt ? { prompt: agentPrompt } : {}), + ...(agentPromptDelivery ? { promptDelivery: agentPromptDelivery } : {}), + ...(agentArgsOverride !== undefined + ? { agentArgs: agentArgsOverride } + : {}), + ...(agentLaunchPreferences + ? { launchPreferences: agentLaunchPreferences } + : {}), + placement: { tabId, leafId }, + presentation: 'background' + }, + agentCreateOperation.clientOperationId + ), + timeoutMs + ), + createEnvironmentId, + connectLifecycleEpoch + ) + const resumeHostAuthorityCapability = resumeProviderSessionToSend + ? agentResumeHostAuthorityCapability(launchAgentToSend) + : undefined + const created = launchAgentToSend + ? agentSessionRequiresHostAuthorityReplay + ? await hostAuthorityCreate() + : await runRemoteAgentSessionLaunch({ + environmentId: createEnvironmentId, + hostAuthority: hostAuthorityCreate, + ...(resumeHostAuthorityCapability + ? { hostAuthorityCapability: resumeHostAuthorityCapability } + : {}), + legacy: legacyCreate + }) + : await legacyCreate() + if (!created) { + if (!destroyed && lifecycleEpoch === connectLifecycleEpoch) { + connecting = false + recovery.markDisconnected() + } + return + } + const createdTerminal = created.terminal + adoptExecutionMetadata(createdTerminal) + if (created.disposition !== undefined && tabId && createdTerminal.tabId) { + recordWebAgentSessionHandoff({ + environmentId: createEnvironmentId, + worktreeId, + provisionalTabId: tabId, + hostTabId: createdTerminal.tabId, + hostTerminalHandle: createdTerminal.handle + }) + // Snapshot parity must not delay attachment to a terminal the host already created. + void refreshWebRuntimeSessionTabsSnapshot(createEnvironmentId, worktreeId, { + expectedEnvironmentPairingRevision: runtimeEnvironmentPairingRevision, + acceptCurrentSnapshot: true, + confirmAgentSessionHandoff: { + provisionalTabId: tabId, + hostTabId: createdTerminal.tabId, + hostTerminalHandle: createdTerminal.handle + } + }) + } + if (destroyed || lifecycleEpoch !== connectLifecycleEpoch) { + if ( + !hostSnapshotOwnsLaunch(created, createEnvironmentId) && + (createdTerminal.handle !== handle || + createEnvironmentId !== currentRuntimeEnvironmentId) + ) { + await closeRemoteTerminal(createdTerminal.handle, createEnvironmentId) + } + return + } + handle = createdTerminal.handle + + if (createdTerminal.isReattach === true) { + storedCallbacks.onReattachDetermined?.() + } + remotePtyId = toRemoteRuntimePtyId(handle, currentRuntimeEnvironmentId) + registerShutdownHandlers(remotePtyId) + connected = true + desiredViewport = { + cols: options.cols ?? 80, + rows: options.rows ?? 24 + } + if (createdTerminal.isReattach !== true) { + onPtySpawn?.(remotePtyId) + } + emitRecoveryState() + + try { + await subscribeToHandle() + } catch (error) { + if (!recoverAfterSubscribeFailure(error, handle, remotePtyId)) { + throw error + } + } + if (destroyed || !connected || !remotePtyId) { + return + } + + return { + id: remotePtyId, + replay: '', + ...(authoritativePtyIncarnationId + ? { incarnationId: authoritativePtyIncarnationId } + : {}), + ...(createdTerminal.isReattach === true ? { isReattach: true } : {}) + } satisfies PtyConnectResult + } catch (error) { + if (!destroyed && lifecycleEpoch === connectLifecycleEpoch) { + connecting = false + const message = runtimeTerminalErrorMessage(error) + if (isRemoteTerminalGoneMessage(message)) { + recovery.cancel() + handleRemoteTerminalError(error) + } else if ( + isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error)) + ) { + scheduleConnectRetryAfterRecoverableFailure() + } else { + recovery.cancel() + emitRecoveryState() + surfaceErrorMessage(message) + } + } + return undefined + } + }, + + attach(options) { + const attachLifecycleEpoch = ++lifecycleEpoch + const generation = ++attachGeneration + cancelTerminalCreateRetryWait() + recovery.cancel() + resetRecoveryReplacementPolicy() + resetSameHandleEndReuse() + clearPublishedHandleWait() + lastAttachOptions = options + storedCallbacks = options.callbacks + terminalEnded = false + connecting = true + emitRecoveryState(true) + // Why: persisted ids are untrusted cache state; the worktree owner selected this transport and must remain authoritative. + currentRuntimeEnvironmentId = runtimeEnvironmentId + const previousHandle = handle + const previousPtyId = remotePtyId + const nextHandle = getRemoteRuntimeTerminalHandle(options.existingPtyId) + if (previousHandle && previousHandle !== nextHandle) { + // Why: debounced input is scoped by the current terminal handle at flush time. + inputBatcher.clear() + } + const persistedEnvironmentId = getRemoteRuntimePtyEnvironmentId(options.existingPtyId) + handle = nextHandle + unregisterShutdownHandlers(previousPtyId) + connected = false + remotePtyId = null + clearPendingViewportClaim() + closeMultiplexedStream() + if (!nextHandle) { + handle = null + connecting = false + emitRecoveryState() + surfaceErrorMessage('Remote runtime terminal id is invalid.') + return + } + const persistedHandle = nextHandle + void (async () => { + if (isWebTerminalSurfaceTabId(tabId ?? '')) { + await attachHostSessionMirror(options, false, generation, attachLifecycleEpoch) + return + } + if (!tabId || !leafId || !worktreeId) { + await adoptResolvedHostPane( + { + handle: persistedHandle, + tabId: tabId ?? '', + leafId: leafId ?? '', + ptyId: null, + worktreeId + }, + options, + false, + generation + ) + return + } + const resolved = await resolvePersistedHostPane() + if (generation !== attachGeneration || destroyed) { + return + } + if ( + !resolved && + resolvePaneUnavailable && + persistedEnvironmentId === currentRuntimeEnvironmentId + ) { + await adoptResolvedHostPane( + { + handle: persistedHandle, + tabId: tabId ?? '', + leafId: leafId ?? '', + ptyId: null, + worktreeId + }, + options, + false, + generation + ) + return + } + if (!resolved) { + surfaceErrorMessage('Remote terminal was closed.') + return + } + await adoptResolvedHostPane(resolved, options, false, generation) + })().catch((error) => { + if ( + generation !== attachGeneration || + attachLifecycleEpoch !== lifecycleEpoch || + destroyed + ) { + return + } + clearPendingViewportClaim() + recovery.cancel() + handleRemoteTerminalError(error) + }) + }, + + disconnect() { + lifecycleEpoch += 1 + attachGeneration += 1 + cancelTerminalCreateRetryWait() + recovery.cancel() + resetRecoveryReplacementPolicy() + resetSameHandleEndReuse() + clearPublishedHandleWait() + inputBatcher.flush() + inputBatcher.clear() + viewportBatcher.flush() + outputProcessor.clearAccumulatedState() + if (!connected && !handle) { + return + } + connected = false + connecting = false + terminalEnded = true + clearPendingViewportClaim() + const id = remotePtyId + unregisterShutdownHandlers(id) + closeMultiplexedStream() + setAttachmentUnavailable() + handle = null + remotePtyId = null + emitRecoveryState() + storedCallbacks.onDisconnect?.() + if (id) { + // disconnect() tears down only this viewer's stream; it never asks the + // remote host to stop the PTY, so an exit here is unverifiable. + onPtyExit?.(id, -1) + } + }, + + detach() { + // Why first: the successor transport owns the PTY after detach, and the batcher flushes + // below can throw past the census drop — a stranded gauge outlives the transport. + outputProcessor.disposePendingSideEffectGauge() + lifecycleEpoch += 1 + attachGeneration += 1 + cancelTerminalCreateRetryWait() + recovery.cancel() + resetRecoveryReplacementPolicy() + resetSameHandleEndReuse() + clearPublishedHandleWait() + inputBatcher.flush() + inputBatcher.clear() + viewportBatcher.flush() + outputProcessor.clearAccumulatedState() + unregisterShutdownHandlers(remotePtyId) + connected = false + connecting = false + clearPendingViewportClaim() + closeMultiplexedStream() + setAttachmentUnavailable() + emitRecoveryState() + storedCallbacks = {} + }, + + sendInput(data: string): boolean { + if (!connected || !handle || recoveryBlocksIo()) { + return false + } + if (!data) { + return true + } + // Why: literal LF bytes from paste/programmatic input must survive; callers use \r or the enter flag for semantic Enter. + return inputBatcher.push(data) + }, + + // Why: query replies (CPR/DSR/DA/OSC) are read in raw mode with a short timeout; the 8ms debounce would miss it and echo the reply onto the prompt (#7329). + sendInputImmediate(data: string): boolean { + const targetHandle = handle + const targetLifecycleEpoch = lifecycleEpoch + if (!connected || !targetHandle || recoveryBlocksIo()) { + return false + } + if (!data) { + return true + } + // Why: wait behind async validation, but keep the reply as its own host-classifiable write. + if (inputBatcher.hasPendingValidation()) { + inputBatcher.enqueueAfterValidation(() => { + if ( + !connected || + lifecycleEpoch !== targetLifecycleEpoch || + handle !== targetHandle || + recoveryBlocksIo() + ) { + return + } + const pending = inputBatcher.takePending() + if (pending) { + sendUnacknowledgedInput(pending) + } + sendUnacknowledgedInput(data, true) + }) + return true + } + const pending = inputBatcher.takePending() + if (pending && !sendUnacknowledgedInput(pending)) { + return false + } + return sendUnacknowledgedInput(data, true) + }, + + sendInputAccepted: sendInputAcceptedToRuntime, + + claimViewport(cols: number, rows: number): boolean { + if (!connected || !handle) { + return false + } + rememberViewport(cols, rows) + if (recoveryBlocksIo()) { + return true + } + viewportBatcher.clear() + sendViewportUpdate(cols, rows, true) + return true + }, + + setOutputPaused(paused: boolean): boolean { + desiredOutputPaused = paused + if (!connected || !handle) { + return false + } + const supported = getCurrentMultiplexedStream(handle)?.setOutputPaused(paused) === true + storedCallbacks.onOutputPauseChanged?.(paused, supported) + return supported + }, + + resize(cols: number, rows: number, meta): boolean { + if (!connected || !handle) { + return false + } + rememberViewport(cols, rows) + if (recoveryBlocksIo()) { + return true + } + if (meta?.claim) { + viewportBatcher.clear() + sendViewportUpdate(cols, rows, true) + return true + } + // Why: xterm fit emits resize bursts on drag/layout-restore; remote runtimes only need the last viewport per frame. + viewportBatcher.queue(cols, rows) + return true + }, + + isConnected() { + return ( + connected && + !recoveryBlocksIo() && + attachmentReady && + multiplexedStream !== null && + multiplexedStreamHandle === handle + ) + }, + + getRecoveryState, + + // Why: dedup exists to stop one outage spamming the surface; once the user dismisses it, the next occurrence is new information again. + notifyErrorSurfaceDismissed() { + surfacedErrorMessages.clear() + }, + + retryRecovery() { + if ( + !destroyed && + !terminalEnded && + !connected && + isWebTerminalSurfaceTabId(tabId ?? '') && + recovery.currentPhase === 'disconnected' + ) { + recovery.cancel() + if (replayLastTransportEntryPoint()) { + return true + } + } + if ( + !destroyed && + !terminalEnded && + !connected && + !handle && + (terminalCreateNeedsReconciliation || agentSessionRequiresHostAuthorityReplay) && + lastConnectOptions && + recovery.currentPhase === 'disconnected' + ) { + recovery.begin() + void transport.connect(lastConnectOptions) + return true + } + // Why: online/resume fires a parked retry; the button must not be weaker than an event (#12684). + if (!destroyed && !terminalEnded && recovery.currentPhase === 'disconnected') { + if (recovery.retryNow()) { + return true + } + } + if ( + destroyed || + terminalEnded || + !connected || + !handle || + recovery.currentPhase !== 'disconnected' + ) { + return false + } + const recoveryEpoch = recovery.begin() + scheduleResubscribeAfterTransportClose(getRecoveryReplacementPolicy(handle), recoveryEpoch) + return true + }, + + getPtyId() { + return remotePtyId + }, + + getConnectionId() { + return null + }, + + getRuntimeEnvironmentId() { + return currentRuntimeEnvironmentId + }, + + getExecutionHostId() { + return authoritativeExecutionHostId + }, + + getRemotePlatform() { + return authoritativeHostPlatform + }, + + async serializeBuffer(opts) { + if (!connected || !handle) { + return null + } + if (!(await waitForAttachmentReady()) || !handle) { + return null + } + return getCurrentMultiplexedStream(handle)?.serializeBuffer(opts) ?? null + }, + + async serializeBufferOutcome(opts): Promise { + if (!connected || !handle) { + return { + availability: { kind: 'retry-worthy', cause: 'connection-not-ready' }, + snapshot: null + } + } + const stream = getCurrentMultiplexedStream(handle) + if (!stream) { + return { + availability: { kind: 'retry-worthy', cause: 'stream-detached' }, + snapshot: null + } + } + return stream.serializeBufferOutcome(opts) + }, + + destroy() { + destroyed = true + setAttachmentUnavailable() + // Why finally: disconnect runs consumer onDisconnect/onPtyExit callbacks; a throw there + // must not strand the gauge in the very path where teardown already went wrong. + try { + this.disconnect() + } finally { + outputProcessor.disposePendingSideEffectGauge() + } + recovery.dispose() + inputBatcher.clear() + viewportBatcher.clear() + } + } + return transport +}