From fc019f348bcb23f4d4c3c99d4f99ceb307339afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 25 Jul 2026 09:23:28 +0100 Subject: [PATCH 01/43] Harden preview automation timeouts and degraded snapshots --- apps/desktop/src/preview/Manager.test.ts | 191 ++++++++++++++++++ apps/desktop/src/preview/Manager.ts | 147 +++++++++++--- apps/server/src/mcp/McpHttpServer.test.ts | 26 ++- apps/server/src/mcp/McpHttpServer.ts | 29 ++- apps/server/src/mcp/toolkits/preview/tools.ts | 2 +- .../preview/PreviewAutomationHosts.tsx | 30 +++ .../preview/previewAutomationErrors.ts | 41 ++++ .../previewAutomationRequestConsumer.test.ts | 97 +++++++++ .../previewAutomationRequestConsumer.ts | 89 +++++--- packages/contracts/src/previewAutomation.ts | 14 +- 10 files changed, 589 insertions(+), 77 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index f1215ee7b60..77b4de4d219 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -20,6 +20,7 @@ import * as BrowserSession from "./BrowserSession.ts"; import * as PreviewManager from "./Manager.ts"; const { + createFromBuffer, createFromPath, fromId, getFocusedWebContents, @@ -29,6 +30,7 @@ const { writeFile, writeImage, } = vi.hoisted(() => ({ + createFromBuffer: vi.fn(), createFromPath: vi.fn((): { readonly isEmpty: () => boolean } => ({ isEmpty: () => false })), fromId: vi.fn(() => null), getFocusedWebContents: vi.fn(() => null), @@ -44,6 +46,7 @@ vi.mock("electron", () => ({ writeImage, }, nativeImage: { + createFromBuffer, createFromPath, }, shell: { @@ -116,6 +119,7 @@ describe("PreviewManager", () => { showItemInFolder.mockClear(); writeImage.mockClear(); createFromPath.mockClear(); + createFromBuffer.mockReset(); webviewSend.mockClear(); }); @@ -146,6 +150,193 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "captures automation screenshots through CDP and recovers when capture is unavailable", + () => + withManager((manager) => + Effect.gen(function* () { + const png = Buffer.from("automation-preview-png"); + const image = { + isEmpty: () => false, + getSize: () => ({ width: 640, height: 360 }), + resize: vi.fn(), + toPNG: () => png, + }; + createFromBuffer.mockReturnValue(image); + let attached = false; + let captureAvailable = true; + const attach = vi.fn(() => { + attached = true; + }); + const detach = vi.fn(() => { + attached = false; + }); + const capturePage = vi.fn(); + const sendCommand = vi.fn( + async (method: string, params?: Record): Promise => { + if (method === "Runtime.evaluate") { + return { + result: { + value: { + url: "https://example.com/", + title: "Example", + loading: false, + visibleText: "Example body", + interactiveElements: [], + }, + }, + }; + } + if (method === "Accessibility.getFullAXTree") { + return { nodes: [] }; + } + if (method === "Page.captureScreenshot") { + if (!captureAvailable) throw new Error("UnknownVizError"); + expect(params).toEqual({ + format: "png", + fromSurface: true, + captureBeyondViewport: false, + }); + return { data: png.toString("base64") }; + } + return undefined; + }, + ); + fromId.mockReturnValue({ + id: 42, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => "https://example.com/", + getTitle: () => "Example", + isLoading: () => false, + isDevToolsOpened: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + capturePage, + debugger: { + isAttached: () => attached, + attach, + detach, + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + } as never); + + yield* manager.createTab("tab_snapshot"); + yield* manager.registerWebview("tab_snapshot", 42); + yield* Effect.yieldNow; + + const captured = yield* manager.automationSnapshot("tab_snapshot"); + + expect(captured).toMatchObject({ + url: "https://example.com/", + title: "Example", + visibleText: "Example body", + accessibilityTree: { nodes: [] }, + screenshot: { + mimeType: "image/png", + data: png.toString("base64"), + width: 640, + height: 360, + }, + }); + expect(capturePage).not.toHaveBeenCalled(); + + captureAvailable = false; + const degraded = yield* manager.automationSnapshot("tab_snapshot"); + expect(degraded.screenshot).toBeNull(); + expect(detach).toHaveBeenCalledOnce(); + + captureAvailable = true; + const recovered = yield* manager.automationSnapshot("tab_snapshot"); + expect(recovered.screenshot).toMatchObject({ width: 640, height: 360 }); + expect(attach).toHaveBeenCalledTimes(2); + }), + ), + ); + + effectIt.effect("releases and resets timed-out automation control sessions", () => + withManager((manager) => + Effect.gen(function* () { + let attached = false; + let firstEvaluation = true; + const attach = vi.fn(() => { + attached = true; + }); + const detach = vi.fn(() => { + attached = false; + }); + const sendCommand = vi.fn(async (method: string): Promise => { + if (method !== "Runtime.evaluate") return undefined; + if (firstEvaluation) { + firstEvaluation = false; + return await new Promise(() => undefined); + } + return { result: { value: "recovered" } }; + }); + fromId.mockReturnValue({ + id: 43, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => "https://example.com/", + getTitle: () => "Example", + isLoading: () => false, + isDevToolsOpened: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => attached, + attach, + detach, + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + } as never); + + yield* manager.createTab("tab_timeout"); + yield* manager.registerWebview("tab_timeout", 43); + yield* Effect.yieldNow; + + const evaluation = yield* manager + .automationEvaluate("tab_timeout", { expression: "document.title" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + yield* TestClock.adjust(15_000); + const timedOut = yield* Effect.exit(Fiber.join(evaluation)); + + expect(Exit.isFailure(timedOut)).toBe(true); + if (Exit.isFailure(timedOut)) { + expect(Option.getOrThrow(Cause.findErrorOption(timedOut.cause))).toMatchObject({ + _tag: "PreviewAutomationTimeoutError", + tabId: "tab_timeout", + }); + } + expect(detach).toHaveBeenCalledOnce(); + + expect( + yield* manager.automationEvaluate("tab_timeout", { + expression: "document.title", + }), + ).toBe("recovered"); + expect(attach).toHaveBeenCalledTimes(2); + }), + ), + ); + effectIt.effect("isolates failed state listeners and continues delivery", () => { const loggedErrors: Array = []; const logger = Logger.make(({ message }) => { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 6c942d4ccb9..4e8c7396c53 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -101,6 +101,9 @@ const MAX_EVALUATION_BYTES = 64_000; const MAX_VISIBLE_TEXT_LENGTH = 20_000; const MAX_INTERACTIVE_ELEMENTS = 200; const MAX_SCREENSHOT_WIDTH = 1280; +const DEFAULT_AUTOMATION_TIMEOUT_MS = 15_000; +const AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS = 250; +const AUTOMATION_SCREENSHOT_TIMEOUT_MS = 5_000; const DIAGNOSTIC_BUFFER_LIMIT = 200; const MAX_ARTIFACT_SITE_SLUG_LENGTH = 80; const AGENT_CURSOR_MOVE_MS = 160; @@ -877,6 +880,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc: Electron.WebContents, action: string, use: (send: SendCommand, sendCleanup: SendCommand) => Effect.Effect, + timeoutMs = DEFAULT_AUTOMATION_TIMEOUT_MS, ) { const sequence = yield* nextCounter(actionSequenceRef); const startedAt = yield* currentIso; @@ -966,7 +970,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const tabs = yield* SynchronizedRef.get(tabsRef); if (tabs.has(tabId)) yield* update(tabId, { controller: "none" }); }); - return yield* control.semaphore.withPermit(execute().pipe(Effect.onExit(finalize))); + const boundedExecution = Effect.gen(function* () { + const result = yield* control.semaphore + .withPermit(execute()) + .pipe(Effect.timeoutOption(Math.max(1, timeoutMs - AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS))); + if (Option.isNone(result)) { + return yield* new PreviewAutomationTimeoutError({ tabId, timeoutMs }); + } + return result.value; + }); + return yield* boundedExecution.pipe( + Effect.onExit(finalize), + Effect.tapError((error) => + isPreviewAutomationTimeoutError(error) ? detachControlSession(wc.id) : Effect.void, + ), + ); }); const evaluateWithDebugger = ( @@ -1911,6 +1929,60 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }; }); + const captureAutomationScreenshot = Effect.fn("PreviewManager.captureAutomationScreenshot")( + function* (tabId: string, wc: Electron.WebContents, send: SendCommand) { + const response = yield* send("Page.captureScreenshot", { + format: "png", + fromSurface: true, + captureBeyondViewport: false, + }); + const data = + typeof response === "object" && + response !== null && + "data" in response && + typeof response.data === "string" && + response.data.length > 0 + ? response.data + : null; + if (data === null) { + return yield* new PreviewOperationError({ + operation: "automationSnapshot.decodeScreenshot", + tabId, + webContentsId: wc.id, + cause: new Error("Page.captureScreenshot returned no PNG data"), + }); + } + const sourceImage = yield* attempt( + { + operation: "automationSnapshot.createImage", + tabId, + webContentsId: wc.id, + }, + () => nativeImage.createFromBuffer(Buffer.from(data, "base64")), + ); + if (sourceImage.isEmpty()) { + return yield* new PreviewOperationError({ + operation: "automationSnapshot.createImage", + tabId, + webContentsId: wc.id, + cause: new Error("Page.captureScreenshot returned an invalid PNG"), + }); + } + const sourceSize = sourceImage.getSize(); + const image = + sourceSize.width > MAX_SCREENSHOT_WIDTH + ? sourceImage.resize({ width: MAX_SCREENSHOT_WIDTH }) + : sourceImage; + const size = image.getSize(); + return { + mimeType: "image/png" as const, + data: image.toPNG().toString("base64"), + width: size.width, + height: size.height, + }; + }, + ); + const captureAutomationSnapshot = Effect.fn("PreviewManager.captureAutomationSnapshot")( function* (tabId: string, wc: Electron.WebContents, send: SendCommand) { yield* Effect.all([send("Runtime.enable"), send("Accessibility.enable")], { @@ -1979,25 +2051,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function })()`, true, ); - const [accessibility, sourceImage, diagnostics, timelines] = yield* Effect.all([ + const [accessibility, diagnostics, timelines] = yield* Effect.all([ send("Accessibility.getFullAXTree"), - attemptPromise( - { - operation: "automationSnapshot.capturePage", - tabId, - webContentsId: wc.id, - }, - () => wc.capturePage(), - ), Ref.get(diagnosticsRef), Ref.get(actionTimelineRef), ]); - const sourceSize = sourceImage.getSize(); - const image = - sourceSize.width > MAX_SCREENSHOT_WIDTH - ? sourceImage.resize({ width: MAX_SCREENSHOT_WIDTH }) - : sourceImage; - const size = image.getSize(); + const screenshotResult = yield* captureAutomationScreenshot(tabId, wc, send).pipe( + Effect.timeoutOption(AUTOMATION_SCREENSHOT_TIMEOUT_MS), + Effect.exit, + ); + const screenshot = + Exit.isSuccess(screenshotResult) && Option.isSome(screenshotResult.value) + ? screenshotResult.value.value + : null; + if (screenshot === null) { + const failure = Exit.isFailure(screenshotResult) + ? screenshotResult.cause + : new PreviewAutomationTimeoutError({ + tabId, + timeoutMs: AUTOMATION_SCREENSHOT_TIMEOUT_MS, + }); + yield* Effect.logWarning("Preview automation screenshot capture was unavailable.", { + tabId, + webContentsId: wc.id, + cause: failure, + }); + // A timed-out debugger command may still settle after its Effect has + // been interrupted. Detach the session so later automation starts + // from a fresh CDP connection instead of inheriting that command. + yield* detachControlSession(wc.id); + } const browserDiagnostics = diagnostics.get(wc.id); return { ...page, @@ -2005,12 +2088,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function consoleEntries: [...(browserDiagnostics?.consoleEntries ?? [])], networkEntries: [...(browserDiagnostics?.networkEntries ?? [])], actionTimeline: [...(timelines.get(tabId) ?? [])], - screenshot: { - mimeType: "image/png" as const, - data: image.toPNG().toString("base64"), - width: size.width, - height: size.height, - }, + screenshot, }; }, ); @@ -2153,8 +2231,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function input: PreviewAutomationClickInput, ) { const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "click", (send) => - performAutomationClick(tabId, input, send), + yield* withControlSession( + tabId, + wc, + "click", + (send) => performAutomationClick(tabId, input, send), + input.timeoutMs, ); }); @@ -2279,8 +2361,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function input: PreviewAutomationTypeInput, ) { const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "type", (send) => - performAutomationType(tabId, input, send), + yield* withControlSession( + tabId, + wc, + "type", + (send) => performAutomationType(tabId, input, send), + input.timeoutMs, ); }); @@ -2504,8 +2590,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function input: PreviewAutomationWaitForInput, ) { const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "waitFor", (send) => - performAutomationWaitFor(tabId, input, send), + yield* withControlSession( + tabId, + wc, + "waitFor", + (send) => performAutomationWaitFor(tabId, input, send), + input.timeoutMs, ); }); @@ -2874,6 +2964,7 @@ export const isPreviewAutomationEvaluationError = Schema.is(PreviewAutomationEva export const isPreviewAutomationInvalidSelectorError = Schema.is( PreviewAutomationInvalidSelectorError, ); +export const isPreviewAutomationTimeoutError = Schema.is(PreviewAutomationTimeoutError); export class PreviewManager extends Context.Service< PreviewManager, diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index ca3341be7f3..d4bce8293ef 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -16,6 +16,7 @@ const environmentId = EnvironmentId.make("environment-mcp-test"); const threadId = ThreadId.make("thread-mcp-test"); const tabId = PreviewTabId.make("tab-mcp-test"); const alternateTabId = PreviewTabId.make("tab-mcp-alternate"); +const noScreenshotTabId = PreviewTabId.make("tab-mcp-no-screenshot"); const invocation = { environmentId, threadId, @@ -183,12 +184,15 @@ it.effect("registers annotated tools and preserves authenticated request context consoleEntries: [], networkEntries: [], actionTimeline: [], - screenshot: { - mimeType: "image/png", - data: Buffer.from("png").toString("base64"), - width: 10, - height: 5, - }, + screenshot: + event.request.tabId === noScreenshotTabId + ? null + : { + mimeType: "image/png", + data: Buffer.from("png").toString("base64"), + width: 10, + height: 5, + }, } : event.request.operation === "press" ? undefined @@ -258,6 +262,16 @@ it.effect("registers annotated tools and preserves authenticated request context alternateTabId, ); + const snapshotWithoutImage = yield* server + .callTool({ name: "preview_snapshot", arguments: { tabId: noScreenshotTabId } }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + expect(snapshotWithoutImage.isError).toBe(false); + expect(snapshotWithoutImage.content.some((content) => content.type === "image")).toBe(false); + expect(snapshotWithoutImage.structuredContent).toMatchObject({ screenshot: null }); + const press = yield* server .callTool({ name: "preview_press", arguments: { key: "Enter" } }) .pipe( diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 6774731a73e..5fc438a404d 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -157,7 +157,7 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot onFailure: previewSnapshotFailure, onSuccess: ({ encodedResult }) => { const snapshot = encodedResult as { - readonly screenshot: { + readonly screenshot: null | { readonly mimeType: "image/png"; readonly data: string; readonly width: number; @@ -168,11 +168,14 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot const { screenshot, ...page } = snapshot; const metadata = { ...page, - screenshot: { - mimeType: screenshot.mimeType, - width: screenshot.width, - height: screenshot.height, - }, + screenshot: + screenshot === null + ? null + : { + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, + }, }; return Effect.succeed( new McpSchema.CallToolResult({ @@ -180,11 +183,15 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot structuredContent: metadata, content: [ { type: "text", text: JSON.stringify(metadata) }, - { - type: "image", - data: new Uint8Array(Buffer.from(screenshot.data, "base64")), - mimeType: screenshot.mimeType, - }, + ...(screenshot === null + ? [] + : [ + { + type: "image" as const, + data: new Uint8Array(Buffer.from(screenshot.data, "base64")), + mimeType: screenshot.mimeType, + }, + ]), ], }), ); diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index d2527fdfb39..92e431b59c1 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -104,7 +104,7 @@ export const PreviewSetAppearanceTool = safeBrowserTool( export const PreviewSnapshotTool = readonlyBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot.", + "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot when capture is available.", parameters: PreviewAutomationTabTargetInput, success: PreviewAutomationSnapshot, failure: PreviewAutomationError, diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 27383b29f19..1f0eaa071c2 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -51,6 +51,7 @@ import { PreviewAutomationOverlayTimeoutError, PreviewAutomationRecordingNotActiveError, PreviewAutomationTargetUnavailableError, + PreviewAutomationVisibilityTimeoutError, PreviewAutomationViewportTimeoutError, } from "./previewAutomationErrors"; import { previewAutomationOpenNeedsOverlay } from "./previewAutomationOpenReadiness"; @@ -86,6 +87,27 @@ const waitForDesktopOverlay = async ( }); }; +const waitForBrowserSurfaceVisibility = async ( + threadRef: ScopedThreadRef, + requestId: string, + tabId: string, + requestTimeoutMs: number, +): Promise => { + const timeoutMs = Math.max(1, Math.min(2_000, requestTimeoutMs - 250)); + const deadline = Date.now() + timeoutMs; + while (Date.now() <= deadline) { + if (useBrowserSurfaceStore.getState().byTabId[tabId]?.visible) return; + await new Promise((resolve) => window.setTimeout(resolve, 50)); + } + throw new PreviewAutomationVisibilityTimeoutError({ + requestId, + environmentId: threadRef.environmentId, + threadId: threadRef.threadId, + tabId, + timeoutMs, + }); +}; + const waitForNavigationReadiness = async ( threadRef: ScopedThreadRef, requestId: string, @@ -399,6 +421,14 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) request.timeoutMs, ); } + if (input.show ?? true) { + await waitForBrowserSurfaceVisibility( + threadRef, + request.requestId, + activeTabId, + request.timeoutMs, + ); + } return await currentStatus(threadRef, activeTabId); } case "navigate": { diff --git a/apps/web/src/components/preview/previewAutomationErrors.ts b/apps/web/src/components/preview/previewAutomationErrors.ts index dcf35de53f2..63b8767ac75 100644 --- a/apps/web/src/components/preview/previewAutomationErrors.ts +++ b/apps/web/src/components/preview/previewAutomationErrors.ts @@ -75,6 +75,45 @@ export class PreviewAutomationViewportTimeoutError extends Schema.TaggedErrorCla } } +export class PreviewAutomationVisibilityTimeoutError extends Schema.TaggedErrorClass()( + "PreviewAutomationVisibilityTimeoutError", + { + requestId: TrimmedNonEmptyString, + environmentId: EnvironmentId, + threadId: ThreadId, + tabId: PreviewTabId, + timeoutMs: Schema.Int, + }, +) { + get responseTag() { + return "PreviewAutomationTimeoutError" as const; + } + + override get message(): string { + return `Preview browser surface for request ${this.requestId} on environment ${this.environmentId} thread ${this.threadId} tab ${this.tabId} did not become visible within ${this.timeoutMs}ms.`; + } +} + +export class PreviewAutomationHostDeadlineExceededError extends Schema.TaggedErrorClass()( + "PreviewAutomationHostDeadlineExceededError", + { + requestId: TrimmedNonEmptyString, + operation: PreviewAutomationOperation, + environmentId: EnvironmentId, + threadId: ThreadId, + tabId: Schema.NullOr(PreviewTabId), + timeoutMs: Schema.Int, + }, +) { + get responseTag() { + return "PreviewAutomationTimeoutError" as const; + } + + override get message(): string { + return `Preview automation ${this.operation} request ${this.requestId} on environment ${this.environmentId} thread ${this.threadId} (tab ${this.tabId ?? "unassigned"}) did not complete within the ${this.timeoutMs}ms host response budget.`; + } +} + export class PreviewAutomationTargetUnavailableError extends Schema.TaggedErrorClass()( "PreviewAutomationTargetUnavailableError", { @@ -209,6 +248,8 @@ export const PreviewAutomationHostError = Schema.Union([ PreviewAutomationOverlayTimeoutError, PreviewAutomationNavigationTimeoutError, PreviewAutomationViewportTimeoutError, + PreviewAutomationVisibilityTimeoutError, + PreviewAutomationHostDeadlineExceededError, PreviewAutomationTargetUnavailableError, PreviewAutomationRecordingNotActiveError, PreviewAutomationTargetNotEditableHostError, diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts index af3a95c32c7..1a86a60201f 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts @@ -10,8 +10,10 @@ import { AsyncResult, Atom, AtomRegistry } from "effect/unstable/reactivity"; import { describe, expect, it, vi } from "vite-plus/test"; import { + PreviewAutomationHostDeadlineExceededError, PreviewAutomationRecordingNotActiveError, PreviewAutomationTargetUnavailableError, + PreviewAutomationVisibilityTimeoutError, PreviewAutomationViewportTimeoutError, } from "./previewAutomationErrors"; import { @@ -291,6 +293,101 @@ describe("previewAutomationRequestConsumer", () => { }); }); + it("preserves browser visibility timeouts as timeout responses", () => { + const error = new PreviewAutomationVisibilityTimeoutError({ + requestId: "request-open", + environmentId, + threadId, + tabId, + timeoutMs: 2_000, + }); + + expect( + serializePreviewAutomationError(error, { + requestId: "request-open", + operation: "open", + environmentId, + threadId, + tabId, + }), + ).toMatchObject({ + _tag: "PreviewAutomationTimeoutError", + detail: { tabId: "tab-1", timeoutMs: 2_000 }, + }); + }); + + it("preserves host response deadline errors as timeout responses", () => { + const error = new PreviewAutomationHostDeadlineExceededError({ + requestId: "request-snapshot", + operation: "snapshot", + environmentId, + threadId, + tabId, + timeoutMs: 14_750, + }); + + expect( + serializePreviewAutomationError(error, { + requestId: "request-snapshot", + operation: "snapshot", + environmentId, + threadId, + tabId, + }), + ).toMatchObject({ + _tag: "PreviewAutomationTimeoutError", + detail: { tabId: "tab-1", timeoutMs: 14_750 }, + }); + }); + + it("responds before the broker deadline when the host operation stalls", async () => { + vi.useFakeTimers(); + try { + const requestsAtom = Atom.make>( + AsyncResult.initial(false), + ); + const respond = vi.fn(async (_response: PreviewAutomationResponse) => undefined); + const state = consumerState(() => new Promise(() => undefined)); + const consumerAtom = createPreviewAutomationRequestConsumerAtom({ + requestsAtom, + clientId, + connectionAtom: state.connectionAtom, + environmentId, + requestHandlerAtom: state.requestHandlerAtom, + respond, + label: "test:preview-automation-host-deadline", + }); + const registry = AtomRegistry.make(); + registry.mount(consumerAtom); + registry.set( + requestsAtom, + AsyncResult.success( + requestEvent("request-stalled", { + operation: "snapshot", + tabId, + timeoutMs: 1_000, + }), + ), + ); + + await vi.advanceTimersByTimeAsync(750); + + expect(respond).toHaveBeenCalledWith( + expect.objectContaining({ + requestId: "request-stalled", + ok: false, + error: expect.objectContaining({ + _tag: "PreviewAutomationTimeoutError", + detail: expect.objectContaining({ timeoutMs: 750 }), + }), + }), + ); + registry.dispose(); + } finally { + vi.useRealTimers(); + } + }); + it("maps desktop non-editable targets to the public typed response", () => { expect( serializePreviewAutomationError( diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts index 89a9387e4af..3c508886d5b 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts @@ -7,6 +7,7 @@ import type { import { AsyncResult, Atom } from "effect/unstable/reactivity"; import { + PreviewAutomationHostDeadlineExceededError, PreviewAutomationOperationError, type PreviewAutomationOperationContext, serializePreviewAutomationHostError, @@ -14,6 +15,46 @@ import { type AutomationStreamResult = AsyncResult.AsyncResult; +const PREVIEW_AUTOMATION_RESPONSE_GRACE_MS = 250; + +const handleWithinResponseBudget = ( + request: PreviewAutomationRequest, + environmentId: PreviewAutomationHost["environmentId"], + handle: Promise, +): Promise => + new Promise((resolve, reject) => { + let settled = false; + const timeoutMs = Math.max(1, request.timeoutMs - PREVIEW_AUTOMATION_RESPONSE_GRACE_MS); + const timer = globalThis.setTimeout(() => { + if (settled) return; + settled = true; + reject( + new PreviewAutomationHostDeadlineExceededError({ + requestId: request.requestId, + operation: request.operation, + environmentId, + threadId: request.threadId, + tabId: request.tabId ?? null, + timeoutMs, + }), + ); + }, timeoutMs); + handle.then( + (value) => { + if (settled) return; + settled = true; + globalThis.clearTimeout(timer); + resolve(value); + }, + (error: unknown) => { + if (settled) return; + settled = true; + globalThis.clearTimeout(timer); + reject(error); + }, + ); + }); + export function serializePreviewAutomationError( error: unknown, context: PreviewAutomationOperationContext, @@ -63,33 +104,31 @@ export function createPreviewAutomationRequestConsumerAtom(options: { return; } const request = event.request; - void get - .once(options.requestHandlerAtom) - .handle(request) - .then( - (value) => - options.respond({ - clientId: options.clientId, - connectionId: event.connectionId, - requestId: request.requestId, - ok: true, - ...(value === undefined ? {} : { result: value }), - }), - (error) => - options.respond({ - clientId: options.clientId, - connectionId: event.connectionId, + const handle = get.once(options.requestHandlerAtom).handle(request); + void handleWithinResponseBudget(request, options.environmentId, handle).then( + (value) => + options.respond({ + clientId: options.clientId, + connectionId: event.connectionId, + requestId: request.requestId, + ok: true, + ...(value === undefined ? {} : { result: value }), + }), + (error) => + options.respond({ + clientId: options.clientId, + connectionId: event.connectionId, + requestId: request.requestId, + ok: false, + error: serializePreviewAutomationError(error, { requestId: request.requestId, - ok: false, - error: serializePreviewAutomationError(error, { - requestId: request.requestId, - operation: request.operation, - environmentId: options.environmentId, - threadId: request.threadId, - tabId: request.tabId ?? null, - }), + operation: request.operation, + environmentId: options.environmentId, + threadId: request.threadId, + tabId: request.tabId ?? null, }), - ); + }), + ); }; get.addFinalizer(() => { diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index f05623cbc99..68aec37252e 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -535,12 +535,14 @@ export const PreviewAutomationSnapshot = Schema.Struct({ consoleEntries: Schema.Array(PreviewAutomationConsoleEntry), networkEntries: Schema.Array(PreviewAutomationNetworkEntry), actionTimeline: Schema.Array(PreviewAutomationActionEvent), - screenshot: Schema.Struct({ - mimeType: Schema.Literal("image/png"), - data: Schema.String, - width: Schema.Int, - height: Schema.Int, - }), + screenshot: Schema.NullOr( + Schema.Struct({ + mimeType: Schema.Literal("image/png"), + data: Schema.String, + width: Schema.Int, + height: Schema.Int, + }), + ), }); export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type; From 7ffae428d407b7375c215a0dfe0e30471fa76eff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 25 Jul 2026 09:35:21 +0100 Subject: [PATCH 02/43] Acknowledge newly created preview tabs before readiness waits --- apps/server/src/mcp/toolkits/preview/tools.ts | 2 +- .../preview/PreviewAutomationHosts.tsx | 12 +++-- .../previewAutomationOpenReadiness.test.ts | 45 +++++++++++++------ .../preview/previewAutomationOpenReadiness.ts | 24 ++++++++-- packages/contracts/src/previewAutomation.ts | 2 +- 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 92e431b59c1..b5342b55caa 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -54,7 +54,7 @@ export const PreviewStatusTool = Tool.make("preview_status", { export const PreviewOpenTool = browserTool( Tool.make("preview_open", { description: - "Show and initialize a collaborative browser tab. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.", + "Show and initialize a collaborative browser tab. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab. Newly created tabs return after creation; wait on the returned tab before interacting while its initial page loads.", parameters: PreviewAutomationOpenInput, success: PreviewAutomationStatus, failure: PreviewAutomationError, diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 1f0eaa071c2..0903d5790fb 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -54,7 +54,7 @@ import { PreviewAutomationVisibilityTimeoutError, PreviewAutomationViewportTimeoutError, } from "./previewAutomationErrors"; -import { previewAutomationOpenNeedsOverlay } from "./previewAutomationOpenReadiness"; +import { resolvePreviewAutomationOpenWaitPolicy } from "./previewAutomationOpenReadiness"; import { createPreviewAutomationRequestConsumerAtom } from "./previewAutomationRequestConsumer"; import { createPreviewAutomationClientId } from "./previewAutomationClientId"; import { @@ -403,7 +403,13 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) if (input.show ?? true) { useRightPanelStore.getState().openBrowser(threadRef, activeTabId); } - if (activeSnapshot && previewAutomationOpenNeedsOverlay(input, activeSnapshot)) { + const waitPolicy = activeSnapshot + ? resolvePreviewAutomationOpenWaitPolicy(input, activeSnapshot, reusedExistingTab) + : null; + if (waitPolicy?.acknowledgeAfterCreation) { + return await currentStatus(threadRef, activeTabId); + } + if (waitPolicy?.waitForOverlay) { await waitForDesktopOverlay( threadRef, request.requestId, @@ -421,7 +427,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) request.timeoutMs, ); } - if (input.show ?? true) { + if (waitPolicy?.waitForVisibility) { await waitForBrowserSurfaceVisibility( threadRef, request.requestId, diff --git a/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts b/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts index 90de86f799d..9d959a2efac 100644 --- a/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts +++ b/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts @@ -1,7 +1,7 @@ import type { PreviewAutomationOpenInput, PreviewSessionSnapshot } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; -import { previewAutomationOpenNeedsOverlay } from "./previewAutomationOpenReadiness"; +import { resolvePreviewAutomationOpenWaitPolicy } from "./previewAutomationOpenReadiness"; const snapshot = (navStatus: PreviewSessionSnapshot["navStatus"]): PreviewSessionSnapshot => ({ threadId: "thread-1", @@ -13,34 +13,53 @@ const snapshot = (navStatus: PreviewSessionSnapshot["navStatus"]): PreviewSessio }); describe("preview automation open readiness", () => { - it("does not wait for a desktop overlay when opening an empty tab", () => { + it("acknowledges a newly created URL tab before cold renderer readiness", () => { expect( - previewAutomationOpenNeedsOverlay( - {} as PreviewAutomationOpenInput, - snapshot({ _tag: "Idle" }), + resolvePreviewAutomationOpenWaitPolicy( + { url: "https://example.com" } as PreviewAutomationOpenInput, + snapshot({ + _tag: "Loading", + url: "https://example.com/", + title: "", + }), + false, ), - ).toBe(false); + ).toEqual({ + acknowledgeAfterCreation: true, + waitForOverlay: false, + waitForVisibility: false, + }); }); - it("waits when an empty tab is immediately given a URL", () => { + it("waits for the overlay and visibility when navigating a reused tab", () => { expect( - previewAutomationOpenNeedsOverlay( + resolvePreviewAutomationOpenWaitPolicy( { url: "https://example.com" } as PreviewAutomationOpenInput, snapshot({ _tag: "Idle" }), + true, ), - ).toBe(true); + ).toEqual({ + acknowledgeAfterCreation: false, + waitForOverlay: true, + waitForVisibility: true, + }); }); - it("waits for existing tabs that already have rendered content", () => { + it("waits for an existing rendered overlay without requiring visibility when show is false", () => { expect( - previewAutomationOpenNeedsOverlay( - {} as PreviewAutomationOpenInput, + resolvePreviewAutomationOpenWaitPolicy( + { show: false } as PreviewAutomationOpenInput, snapshot({ _tag: "Success", url: "https://example.com/", title: "Example", }), + true, ), - ).toBe(true); + ).toEqual({ + acknowledgeAfterCreation: false, + waitForOverlay: true, + waitForVisibility: false, + }); }); }); diff --git a/apps/web/src/components/preview/previewAutomationOpenReadiness.ts b/apps/web/src/components/preview/previewAutomationOpenReadiness.ts index 416c2f87c64..ca637031d17 100644 --- a/apps/web/src/components/preview/previewAutomationOpenReadiness.ts +++ b/apps/web/src/components/preview/previewAutomationOpenReadiness.ts @@ -1,8 +1,26 @@ import type { PreviewAutomationOpenInput, PreviewSessionSnapshot } from "@t3tools/contracts"; -export function previewAutomationOpenNeedsOverlay( +export interface PreviewAutomationOpenWaitPolicy { + readonly acknowledgeAfterCreation: boolean; + readonly waitForOverlay: boolean; + readonly waitForVisibility: boolean; +} + +export function resolvePreviewAutomationOpenWaitPolicy( input: PreviewAutomationOpenInput, snapshot: PreviewSessionSnapshot, -): boolean { - return input.url !== undefined || snapshot.navStatus._tag !== "Idle"; + reusedExistingTab: boolean, +): PreviewAutomationOpenWaitPolicy { + if (!reusedExistingTab) { + return { + acknowledgeAfterCreation: true, + waitForOverlay: false, + waitForVisibility: false, + }; + } + return { + acknowledgeAfterCreation: false, + waitForOverlay: input.url !== undefined || snapshot.navStatus._tag !== "Idle", + waitForVisibility: input.show ?? true, + }; } diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 68aec37252e..9460ed78287 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -107,7 +107,7 @@ export const PreviewAutomationOpenInput = Schema.Struct({ ) .annotate({ description: - "Opens the collaborative browser for the current thread. Use preview_navigate afterward when readiness waiting matters.", + "Opens the collaborative browser for the current thread. Newly created tabs acknowledge creation before renderer readiness; wait on the returned tab before interacting while its initial page loads.", }); export type PreviewAutomationOpenInput = typeof PreviewAutomationOpenInput.Type; From af718057d0574052ddd81b231311c1718020eb01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 25 Jul 2026 11:38:28 +0100 Subject: [PATCH 03/43] Add background preview screenshot capture --- apps/desktop/src/ipc/methods/preview.ts | 7 +- apps/desktop/src/preload.ts | 7 +- apps/desktop/src/preview/Manager.test.ts | 94 +++++---- apps/desktop/src/preview/Manager.ts | 178 ++++++++++++++---- apps/web/src/browser/HostedBrowserWebview.tsx | 8 +- .../src/browser/browserSurfaceStore.test.ts | 23 ++- apps/web/src/browser/browserSurfaceStore.ts | 33 ++++ .../browser/hostedBrowserWebviewStyle.test.ts | 23 +++ .../src/browser/hostedBrowserWebviewStyle.ts | 17 +- .../preview/PreviewAutomationHosts.tsx | 41 +++- packages/contracts/src/ipc.ts | 7 +- 11 files changed, 358 insertions(+), 80 deletions(-) diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 28405288f6c..97ddd8be0d4 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -4,6 +4,7 @@ import { DesktopPreviewAutomationClickInputSchema, DesktopPreviewAutomationEvaluateInputSchema, DesktopPreviewAutomationPressInputSchema, + DesktopPreviewAutomationSnapshotInputSchema, DesktopPreviewAutomationScrollInputSchema, DesktopPreviewAutomationTypeInputSchema, DesktopPreviewAutomationWaitForInputSchema, @@ -266,11 +267,11 @@ export const automationStatus = DesktopIpc.makeIpcMethod({ export const automationSnapshot = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, - payload: DesktopPreviewTabInputSchema, + payload: DesktopPreviewAutomationSnapshotInputSchema, result: PreviewAutomationSnapshot, - handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId }) { + handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId, background }) { const manager = yield* PreviewManager.PreviewManager; - return yield* manager.automationSnapshot(tabId); + return yield* manager.automationSnapshot(tabId, background); }), }); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 228114fd1d1..035aaef2361 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -203,8 +203,11 @@ contextBridge.exposeInMainWorld("desktopBridge", { automation: { status: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_STATUS_CHANNEL, { tabId }), - snapshot: (tabId) => - ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, { tabId }), + snapshot: (tabId, background) => + ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, { + tabId, + background, + }), click: (tabId, input) => ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_CLICK_CHANNEL, { tabId, input }), type: (tabId, input) => diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 77b4de4d219..88117a2ef0f 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -171,37 +171,32 @@ describe("PreviewManager", () => { const detach = vi.fn(() => { attached = false; }); - const capturePage = vi.fn(); - const sendCommand = vi.fn( - async (method: string, params?: Record): Promise => { - if (method === "Runtime.evaluate") { - return { - result: { - value: { - url: "https://example.com/", - title: "Example", - loading: false, - visibleText: "Example body", - interactiveElements: [], - }, + const focus = vi.fn(); + const restoreFocus = vi.fn(); + const capturePage = vi.fn(async () => image); + const sendCommand = vi.fn(async (method: string): Promise => { + if (method === "Runtime.evaluate") { + return { + result: { + value: { + url: "https://example.com/", + title: "Example", + loading: false, + visibleText: "Example body", + interactiveElements: [], }, - }; - } - if (method === "Accessibility.getFullAXTree") { - return { nodes: [] }; - } - if (method === "Page.captureScreenshot") { - if (!captureAvailable) throw new Error("UnknownVizError"); - expect(params).toEqual({ - format: "png", - fromSurface: true, - captureBeyondViewport: false, - }); - return { data: png.toString("base64") }; - } - return undefined; - }, - ); + }, + }; + } + if (method === "Accessibility.getFullAXTree") { + return { nodes: [] }; + } + if (method === "Page.captureScreenshot") { + if (!captureAvailable) throw new Error("UnknownVizError"); + return { data: png.toString("base64") }; + } + return undefined; + }); fromId.mockReturnValue({ id: 42, isDestroyed: () => false, @@ -218,6 +213,7 @@ describe("PreviewManager", () => { send: webviewSend, navigationHistory: { canGoBack: () => false, canGoForward: () => false }, setWindowOpenHandler: vi.fn(), + focus, capturePage, debugger: { isAttached: () => attached, @@ -228,6 +224,11 @@ describe("PreviewManager", () => { off: vi.fn(), }, } as never); + getFocusedWebContents.mockReturnValue({ + id: 7, + isDestroyed: () => false, + focus: restoreFocus, + } as never); yield* manager.createTab("tab_snapshot"); yield* manager.registerWebview("tab_snapshot", 42); @@ -248,16 +249,45 @@ describe("PreviewManager", () => { }, }); expect(capturePage).not.toHaveBeenCalled(); + expect(sendCommand).toHaveBeenCalledWith("Page.captureScreenshot", { + format: "png", + fromSurface: true, + captureBeyondViewport: false, + }); + + const backgroundCdpCaptured = yield* manager.automationSnapshot("tab_snapshot", true); + expect(backgroundCdpCaptured.screenshot).toMatchObject({ width: 640, height: 360 }); + expect(sendCommand).toHaveBeenCalledWith("Page.captureScreenshot", { + format: "png", + fromSurface: true, + captureBeyondViewport: false, + }); + expect(sendCommand).toHaveBeenCalledWith("Page.bringToFront", undefined); + expect(focus).toHaveBeenCalledOnce(); + expect(restoreFocus).toHaveBeenCalledOnce(); + expect(capturePage).not.toHaveBeenCalled(); captureAvailable = false; + const backgroundFallbackCaptured = yield* manager.automationSnapshot( + "tab_snapshot", + true, + ); + expect(backgroundFallbackCaptured.screenshot).toMatchObject({ width: 640, height: 360 }); + expect(focus).toHaveBeenCalledTimes(2); + expect(restoreFocus).toHaveBeenCalledTimes(2); + expect(capturePage).toHaveBeenCalledOnce(); + expect(capturePage).toHaveBeenCalledWith(undefined, { stayHidden: false }); + + capturePage.mockClear(); const degraded = yield* manager.automationSnapshot("tab_snapshot"); expect(degraded.screenshot).toBeNull(); - expect(detach).toHaveBeenCalledOnce(); + expect(capturePage).not.toHaveBeenCalled(); + expect(detach).toHaveBeenCalledTimes(2); captureAvailable = true; const recovered = yield* manager.automationSnapshot("tab_snapshot"); expect(recovered.screenshot).toMatchObject({ width: 640, height: 360 }); - expect(attach).toHaveBeenCalledTimes(2); + expect(attach).toHaveBeenCalledTimes(3); }), ), ); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 4e8c7396c53..2e64d86a0f7 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -104,6 +104,8 @@ const MAX_SCREENSHOT_WIDTH = 1280; const DEFAULT_AUTOMATION_TIMEOUT_MS = 15_000; const AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS = 250; const AUTOMATION_SCREENSHOT_TIMEOUT_MS = 5_000; +const AUTOMATION_BACKGROUND_CDP_SCREENSHOT_TIMEOUT_MS = 2_000; +const AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS = 3_000; const DIAGNOSTIC_BUFFER_LIMIT = 200; const MAX_ARTIFACT_SITE_SLUG_LENGTH = 80; const AGENT_CURSOR_MOVE_MS = 160; @@ -1929,6 +1931,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }; }); + const encodeAutomationScreenshot = Effect.fn("PreviewManager.encodeAutomationScreenshot")( + function* ( + tabId: string, + wc: Electron.WebContents, + sourceImage: Electron.NativeImage, + operation: string, + ) { + if (sourceImage.isEmpty()) { + return yield* new PreviewOperationError({ + operation, + tabId, + webContentsId: wc.id, + cause: new Error("Screenshot capture returned an invalid PNG"), + }); + } + const sourceSize = sourceImage.getSize(); + const image = + sourceSize.width > MAX_SCREENSHOT_WIDTH + ? sourceImage.resize({ width: MAX_SCREENSHOT_WIDTH }) + : sourceImage; + const size = image.getSize(); + return { + mimeType: "image/png" as const, + data: image.toPNG().toString("base64"), + width: size.width, + height: size.height, + }; + }, + ); + const captureAutomationScreenshot = Effect.fn("PreviewManager.captureAutomationScreenshot")( function* (tabId: string, wc: Electron.WebContents, send: SendCommand) { const response = yield* send("Page.captureScreenshot", { @@ -1960,31 +1992,37 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }, () => nativeImage.createFromBuffer(Buffer.from(data, "base64")), ); - if (sourceImage.isEmpty()) { - return yield* new PreviewOperationError({ - operation: "automationSnapshot.createImage", - tabId, - webContentsId: wc.id, - cause: new Error("Page.captureScreenshot returned an invalid PNG"), - }); - } - const sourceSize = sourceImage.getSize(); - const image = - sourceSize.width > MAX_SCREENSHOT_WIDTH - ? sourceImage.resize({ width: MAX_SCREENSHOT_WIDTH }) - : sourceImage; - const size = image.getSize(); - return { - mimeType: "image/png" as const, - data: image.toPNG().toString("base64"), - width: size.width, - height: size.height, - }; + return yield* encodeAutomationScreenshot( + tabId, + wc, + sourceImage, + "automationSnapshot.createImage", + ); }, ); + const captureBackgroundPage = Effect.fn("PreviewManager.captureBackgroundPage")(function* ( + tabId: string, + wc: Electron.WebContents, + ) { + const sourceImage = yield* attemptPromise( + { + operation: "automationSnapshot.captureBackgroundPage", + tabId, + webContentsId: wc.id, + }, + () => wc.capturePage(undefined, { stayHidden: false }), + ); + return yield* encodeAutomationScreenshot( + tabId, + wc, + sourceImage, + "automationSnapshot.captureBackgroundPage", + ); + }); + const captureAutomationSnapshot = Effect.fn("PreviewManager.captureAutomationSnapshot")( - function* (tabId: string, wc: Electron.WebContents, send: SendCommand) { + function* (tabId: string, wc: Electron.WebContents, send: SendCommand, background: boolean) { yield* Effect.all([send("Runtime.enable"), send("Accessibility.enable")], { concurrency: 2, discard: true, @@ -2056,30 +2094,58 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function Ref.get(diagnosticsRef), Ref.get(actionTimelineRef), ]); - const screenshotResult = yield* captureAutomationScreenshot(tabId, wc, send).pipe( - Effect.timeoutOption(AUTOMATION_SCREENSHOT_TIMEOUT_MS), + const cdpScreenshotTimeoutMs = background + ? AUTOMATION_BACKGROUND_CDP_SCREENSHOT_TIMEOUT_MS + : AUTOMATION_SCREENSHOT_TIMEOUT_MS; + const cdpScreenshotResult = yield* captureAutomationScreenshot(tabId, wc, send).pipe( + Effect.timeoutOption(cdpScreenshotTimeoutMs), Effect.exit, ); - const screenshot = - Exit.isSuccess(screenshotResult) && Option.isSome(screenshotResult.value) - ? screenshotResult.value.value + let screenshot: PreviewAutomationSnapshot["screenshot"] = + Exit.isSuccess(cdpScreenshotResult) && Option.isSome(cdpScreenshotResult.value) + ? cdpScreenshotResult.value.value : null; if (screenshot === null) { - const failure = Exit.isFailure(screenshotResult) - ? screenshotResult.cause + const cdpFailure = Exit.isFailure(cdpScreenshotResult) + ? cdpScreenshotResult.cause : new PreviewAutomationTimeoutError({ tabId, - timeoutMs: AUTOMATION_SCREENSHOT_TIMEOUT_MS, + timeoutMs: cdpScreenshotTimeoutMs, }); - yield* Effect.logWarning("Preview automation screenshot capture was unavailable.", { - tabId, - webContentsId: wc.id, - cause: failure, - }); // A timed-out debugger command may still settle after its Effect has // been interrupted. Detach the session so later automation starts // from a fresh CDP connection instead of inheriting that command. yield* detachControlSession(wc.id); + const backgroundScreenshotResult = background + ? yield* captureBackgroundPage(tabId, wc).pipe( + Effect.timeoutOption(AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS), + Effect.exit, + ) + : null; + screenshot = + backgroundScreenshotResult !== null && + Exit.isSuccess(backgroundScreenshotResult) && + Option.isSome(backgroundScreenshotResult.value) + ? backgroundScreenshotResult.value.value + : null; + if (screenshot === null) { + const backgroundFailure = + backgroundScreenshotResult === null + ? null + : Exit.isFailure(backgroundScreenshotResult) + ? backgroundScreenshotResult.cause + : new PreviewAutomationTimeoutError({ + tabId, + timeoutMs: AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS, + }); + yield* Effect.logWarning("Preview automation screenshot capture was unavailable.", { + tabId, + webContentsId: wc.id, + background, + cdpCause: cdpFailure, + backgroundCause: backgroundFailure, + }); + } } const browserDiagnostics = diagnostics.get(wc.id); return { @@ -2095,11 +2161,50 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const automationSnapshot = Effect.fn("PreviewManager.automationSnapshot")(function* ( tabId: string, + background = false, ) { const wc = yield* requireWebContents(tabId); - return yield* withControlSession(tabId, wc, "snapshot", (send) => - captureAutomationSnapshot(tabId, wc, send), + if (!background) { + return yield* withControlSession(tabId, wc, "snapshot", (send) => + captureAutomationSnapshot(tabId, wc, send, false), + ); + } + + const previouslyFocused = yield* attempt( + { operation: "automationSnapshot.getFocusedWebContents", tabId, webContentsId: wc.id }, + () => webContents.getFocusedWebContents(), ); + const restoreFocus = + previouslyFocused && previouslyFocused.id !== wc.id + ? attempt( + { + operation: "automationSnapshot.restoreFocusedWebContents", + tabId, + webContentsId: previouslyFocused.id, + }, + () => { + if (!previouslyFocused.isDestroyed()) previouslyFocused.focus(); + }, + ).pipe(Effect.ignore) + : Effect.void; + + // A mounted-but-unselected remains a live guest, but Chromium + // does not expose its composited pixels until that guest is foregrounded. + // The renderer stages it transparently while this short capture lease + // activates the guest itself. Restoring the prior WebContents preserves + // both application focus and the user-visible preview selection. + return yield* Effect.gen(function* () { + yield* attempt( + { operation: "automationSnapshot.focusWebContents", tabId, webContentsId: wc.id }, + () => wc.focus(), + ); + return yield* withControlSession(tabId, wc, "snapshot", (send) => + Effect.gen(function* () { + yield* send("Page.bringToFront"); + return yield* captureAutomationSnapshot(tabId, wc, send, true); + }), + ); + }).pipe(Effect.ensuring(restoreFocus)); }); const resolveClickPoint = Effect.fn("PreviewManager.resolveClickPoint")(function* ( @@ -3018,6 +3123,7 @@ export class PreviewManager extends Context.Service< ) => Effect.Effect; readonly automationSnapshot: ( tabId: string, + background?: boolean, ) => Effect.Effect; readonly automationClick: ( tabId: string, diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index a72da49c6fa..b88375fd249 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -52,6 +52,7 @@ export function HostedBrowserWebview(props: { useShallow((state) => { const current = state.byTabId[tabId]; return { + backgroundCapture: (state.backgroundCaptureCountByTabId[tabId] ?? 0) > 0, rect: resolveBrowserSurfacePanelRect(state.byTabId, tabId), visible: current?.visible ?? false, }; @@ -113,6 +114,7 @@ export function HostedBrowserWebview(props: { }, [config, tabId]); const active = presentation.visible && presentation.rect !== null; + const backgroundCapture = !active && presentation.backgroundCapture && presentation.rect !== null; const lastRect = presentation.rect; const normalizedZoomFactor = Number.isFinite(zoomFactor) && zoomFactor > 0 ? zoomFactor : 1; const viewportWidth = viewport._tag === "fill" ? null : viewport.width; @@ -131,7 +133,7 @@ export function HostedBrowserWebview(props: { height: viewport.height * normalizedZoomFactor, } : { width: lastRect?.width ?? 1280, height: lastRect?.height ?? 800 }; - const containerSize = active && lastRect ? lastRect : hiddenSize; + const containerSize = (active || backgroundCapture) && lastRect ? lastRect : hiddenSize; const deviceToolbarVisible = active && viewport._tag !== "fill"; const { activeDrag, @@ -178,6 +180,7 @@ export function HostedBrowserWebview(props: { const wrapperStyle = resolveHostedBrowserWebviewWrapperStyle({ active, + backgroundCapture, rect: lastRect, hiddenSize, }); @@ -189,6 +192,7 @@ export function HostedBrowserWebview(props: { style={{ ...wrapperStyle, overscrollBehavior: "contain" }} onScroll={syncContentPresentation} data-preview-viewport={tabId} + data-preview-background-capture={backgroundCapture ? "true" : undefined} >
{deviceToolbarVisible && effectiveViewport._tag !== "fill" ? ( @@ -219,7 +223,7 @@ export function HostedBrowserWebview(props: { ? Math.max(1, Math.round(layout.viewportHeight / normalizedZoomFactor)) : effectiveViewport.height } - aria-hidden={active ? undefined : true} + aria-hidden={active || backgroundCapture ? undefined : true} className={cn( "absolute flex overflow-hidden bg-background", active && !layout.fillsPanel && "ring-1 ring-border/70 shadow-sm", diff --git a/apps/web/src/browser/browserSurfaceStore.test.ts b/apps/web/src/browser/browserSurfaceStore.test.ts index ecfce8cb432..992c52cc599 100644 --- a/apps/web/src/browser/browserSurfaceStore.test.ts +++ b/apps/web/src/browser/browserSurfaceStore.test.ts @@ -2,13 +2,34 @@ import { beforeEach, describe, expect, it } from "vite-plus/test"; import { acquireBrowserSurface, + acquireBrowserSurfaceBackgroundCapture, resolveBrowserSurfacePanelRect, useBrowserSurfaceStore, } from "./browserSurfaceStore"; describe("browserSurfaceStore", () => { beforeEach(() => { - useBrowserSurfaceStore.setState({ byTabId: {} }); + useBrowserSurfaceStore.setState({ byTabId: {}, backgroundCaptureCountByTabId: {} }); + }); + + it("reference-counts background capture leases", () => { + const releaseFirst = acquireBrowserSurfaceBackgroundCapture("tab-background"); + const releaseSecond = acquireBrowserSurfaceBackgroundCapture("tab-background"); + + expect(useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"]).toBe( + 2, + ); + + releaseFirst(); + releaseFirst(); + expect(useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"]).toBe( + 1, + ); + + releaseSecond(); + expect( + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + ).toBeUndefined(); }); it("tracks content dimensions for a browser that has never been visible", () => { diff --git a/apps/web/src/browser/browserSurfaceStore.ts b/apps/web/src/browser/browserSurfaceStore.ts index 58012a11a30..5ce4d9e78d3 100644 --- a/apps/web/src/browser/browserSurfaceStore.ts +++ b/apps/web/src/browser/browserSurfaceStore.ts @@ -27,6 +27,7 @@ export interface BrowserSurfaceContentPresentation { interface BrowserSurfaceStoreState { readonly byTabId: Record; + readonly backgroundCaptureCountByTabId: Record; readonly claim: (tabId: string, owner: symbol) => void; readonly present: ( tabId: string, @@ -36,6 +37,8 @@ interface BrowserSurfaceStoreState { ) => void; readonly presentContent: (tabId: string, content: BrowserSurfaceContentPresentation) => void; readonly release: (tabId: string, owner: symbol) => void; + readonly retainBackgroundCapture: (tabId: string) => void; + readonly releaseBackgroundCapture: (tabId: string) => void; } export interface BrowserSurfaceLease { @@ -72,6 +75,7 @@ const rectEquals = (left: BrowserSurfaceRect | null, right: BrowserSurfaceRect): export const useBrowserSurfaceStore = create()((set) => ({ byTabId: {}, + backgroundCaptureCountByTabId: {}, claim: (tabId, owner) => set((state) => { const current = state.byTabId[tabId]; @@ -149,6 +153,25 @@ export const useBrowserSurfaceStore = create()((set) = }, }; }), + retainBackgroundCapture: (tabId) => + set((state) => ({ + backgroundCaptureCountByTabId: { + ...state.backgroundCaptureCountByTabId, + [tabId]: (state.backgroundCaptureCountByTabId[tabId] ?? 0) + 1, + }, + })), + releaseBackgroundCapture: (tabId) => + set((state) => { + const current = state.backgroundCaptureCountByTabId[tabId] ?? 0; + if (current <= 0) return state; + const next = { ...state.backgroundCaptureCountByTabId }; + if (current === 1) { + delete next[tabId]; + } else { + next[tabId] = current - 1; + } + return { backgroundCaptureCountByTabId: next }; + }), })); export function acquireBrowserSurface(tabId: string): BrowserSurfaceLease { @@ -168,3 +191,13 @@ export function acquireBrowserSurface(tabId: string): BrowserSurfaceLease { }, }; } + +export function acquireBrowserSurfaceBackgroundCapture(tabId: string): () => void { + let released = false; + useBrowserSurfaceStore.getState().retainBackgroundCapture(tabId); + return () => { + if (released) return; + released = true; + useBrowserSurfaceStore.getState().releaseBackgroundCapture(tabId); + }; +} diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts index 826684bb06f..88aa3629b26 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "vite-plus/test"; import { + BACKGROUND_CAPTURE_BROWSER_WEBVIEW_OPACITY, + BACKGROUND_CAPTURE_BROWSER_WEBVIEW_Z_INDEX, HIDDEN_BROWSER_WEBVIEW_OFFSET, resolveHostedBrowserWebviewWrapperStyle, } from "./hostedBrowserWebviewStyle"; @@ -10,6 +12,7 @@ describe("resolveHostedBrowserWebviewWrapperStyle", () => { expect( resolveHostedBrowserWebviewWrapperStyle({ active: true, + backgroundCapture: false, rect: { x: 12, y: 34, width: 800, height: 600 }, hiddenSize: { width: 1280, height: 800 }, }), @@ -23,9 +26,29 @@ describe("resolveHostedBrowserWebviewWrapperStyle", () => { }); }); + it("places a background capture target beneath the active browser surface", () => { + expect( + resolveHostedBrowserWebviewWrapperStyle({ + active: false, + backgroundCapture: true, + rect: { x: 12, y: 34, width: 800, height: 600 }, + hiddenSize: { width: 393, height: 852 }, + }), + ).toEqual({ + left: 12, + top: 34, + width: 800, + height: 600, + zIndex: BACKGROUND_CAPTURE_BROWSER_WEBVIEW_Z_INDEX, + pointerEvents: "none", + opacity: BACKGROUND_CAPTURE_BROWSER_WEBVIEW_OPACITY, + }); + }); + it("keeps an inactive webview paintable while moving it offscreen", () => { const style = resolveHostedBrowserWebviewWrapperStyle({ active: false, + backgroundCapture: false, rect: { x: 12, y: 34, width: 800, height: 600 }, hiddenSize: { width: 393, height: 852 }, }); diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.ts index 4dade986e1f..f965bebd832 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.ts @@ -12,17 +12,21 @@ export interface HostedBrowserWebviewWrapperStyle { readonly height: number; readonly zIndex: number; readonly pointerEvents: "auto" | "none"; + readonly opacity?: number; readonly visibility?: "visible"; } export const HIDDEN_BROWSER_WEBVIEW_OFFSET = -100_000; +export const BACKGROUND_CAPTURE_BROWSER_WEBVIEW_Z_INDEX = 31; +export const BACKGROUND_CAPTURE_BROWSER_WEBVIEW_OPACITY = 0.001; export function resolveHostedBrowserWebviewWrapperStyle(input: { readonly active: boolean; + readonly backgroundCapture: boolean; readonly rect: BrowserSurfaceRect | null; readonly hiddenSize: HostedBrowserWebviewSize; }): HostedBrowserWebviewWrapperStyle { - const { active, hiddenSize, rect } = input; + const { active, backgroundCapture, hiddenSize, rect } = input; if (active && rect) { return { left: rect.x, @@ -33,6 +37,17 @@ export function resolveHostedBrowserWebviewWrapperStyle(input: { pointerEvents: "auto", }; } + if (backgroundCapture && rect) { + return { + left: rect.x, + top: rect.y, + width: rect.width, + height: rect.height, + zIndex: BACKGROUND_CAPTURE_BROWSER_WEBVIEW_Z_INDEX, + pointerEvents: "none", + opacity: BACKGROUND_CAPTURE_BROWSER_WEBVIEW_OPACITY, + }; + } return { left: HIDDEN_BROWSER_WEBVIEW_OFFSET, diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 0903d5790fb..1322f9ffbd0 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -37,7 +37,10 @@ import { stopBrowserRecording, } from "~/browser/browserRecording"; import { resolveBrowserRecordingStopTarget } from "~/browser/browserRecordingScope"; -import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; +import { + acquireBrowserSurfaceBackgroundCapture, + useBrowserSurfaceStore, +} from "~/browser/browserSurfaceStore"; import { isElectron } from "~/env"; import { useEnvironments } from "~/state/environments"; import { previewEnvironment } from "~/state/preview"; @@ -108,6 +111,28 @@ const waitForBrowserSurfaceVisibility = async ( }); }; +const waitForBackgroundCapturePresentation = async (tabId: string): Promise => { + const deadline = Date.now() + 1_000; + while (Date.now() <= deadline) { + const wrapper = Array.from( + document.querySelectorAll("[data-preview-viewport]"), + ).find( + (candidate) => + candidate.dataset["previewViewport"] === tabId && + candidate.dataset["previewBackgroundCapture"] === "true", + ); + if (wrapper) { + // Force the staged wrapper through layout, then allow Chromium two + // compositor frames before asking the guest WebContents for pixels. + void wrapper.offsetWidth; + await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); + await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); + return; + } + await new Promise((resolve) => window.setTimeout(resolve, 16)); + } +}; + const waitForNavigationReadiness = async ( threadRef: ScopedThreadRef, requestId: string, @@ -500,7 +525,19 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "snapshot": { const ready = await requireReadyTab(); - return await ready.bridge.automation.snapshot(ready.tabId); + const background = !( + useBrowserSurfaceStore.getState().byTabId[ready.tabId]?.visible ?? false + ); + if (!background) { + return await ready.bridge.automation.snapshot(ready.tabId, false); + } + const releaseCapture = acquireBrowserSurfaceBackgroundCapture(ready.tabId); + try { + await waitForBackgroundCapturePresentation(ready.tabId); + return await ready.bridge.automation.snapshot(ready.tabId, true); + } finally { + releaseCapture(); + } } case "click": { const ready = await requireReadyTab(); diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 4b1676d7926..c357c3b5863 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -909,6 +909,11 @@ export const DesktopPreviewTabInputSchema = Schema.Struct({ tabId: DesktopPreviewTabIdSchema, }); +export const DesktopPreviewAutomationSnapshotInputSchema = Schema.Struct({ + tabId: DesktopPreviewTabIdSchema, + background: Schema.Boolean, +}); + export const DesktopPreviewRegisterWebviewInputSchema = Schema.Struct({ tabId: DesktopPreviewTabIdSchema, webContentsId: Schema.Int.check(Schema.isGreaterThan(0)), @@ -1092,7 +1097,7 @@ export interface DesktopPreviewBridge { }; automation: { status: (tabId: string) => Promise; - snapshot: (tabId: string) => Promise; + snapshot: (tabId: string, background: boolean) => Promise; click: (tabId: string, input: PreviewAutomationClickInput) => Promise; type: (tabId: string, input: PreviewAutomationTypeInput) => Promise; press: (tabId: string, input: PreviewAutomationPressInput) => Promise; From 49b88400bf1fb532c4c024da68538ad73f8b31e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 25 Jul 2026 14:03:52 +0100 Subject: [PATCH 04/43] Harden preview automation across background tabs and startup flows --- apps/desktop/src/preview/Manager.test.ts | 11 +- apps/desktop/src/preview/Manager.ts | 55 ++++--- apps/server/src/mcp/toolkits/preview/tools.ts | 2 +- apps/web/src/browser/HostedBrowserWebview.tsx | 19 ++- .../src/browser/browserSurfaceStore.test.ts | 29 ++++ apps/web/src/browser/browserSurfaceStore.ts | 22 +++ .../auth/PairingRouteSurface.logic.test.ts | 17 ++ .../auth/PairingRouteSurface.logic.ts | 8 + .../components/auth/PairingRouteSurface.tsx | 28 ++-- .../preview/PreviewAutomationHosts.tsx | 151 +++++++++++++----- .../preview/previewAutomationErrors.ts | 4 + .../previewAutomationOpenReadiness.test.ts | 20 ++- .../previewAutomationPresentation.test.ts | 60 +++++++ .../preview/previewAutomationPresentation.ts | 19 +++ .../previewAutomationRequestConsumer.test.ts | 13 +- packages/contracts/src/previewAutomation.ts | 2 +- scripts/dev-runner.test.ts | 10 +- scripts/dev-runner.ts | 19 +-- 18 files changed, 393 insertions(+), 96 deletions(-) create mode 100644 apps/web/src/components/auth/PairingRouteSurface.logic.test.ts create mode 100644 apps/web/src/components/auth/PairingRouteSurface.logic.ts create mode 100644 apps/web/src/components/preview/previewAutomationPresentation.test.ts create mode 100644 apps/web/src/components/preview/previewAutomationPresentation.ts diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 88117a2ef0f..b1647188ea5 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -292,11 +292,11 @@ describe("PreviewManager", () => { ), ); - effectIt.effect("releases and resets timed-out automation control sessions", () => + effectIt.effect("bounds debugger initialization and recovers the control session", () => withManager((manager) => Effect.gen(function* () { let attached = false; - let firstEvaluation = true; + let firstInitialization = true; const attach = vi.fn(() => { attached = true; }); @@ -304,12 +304,11 @@ describe("PreviewManager", () => { attached = false; }); const sendCommand = vi.fn(async (method: string): Promise => { - if (method !== "Runtime.evaluate") return undefined; - if (firstEvaluation) { - firstEvaluation = false; + if (method === "Runtime.enable" && firstInitialization) { + firstInitialization = false; return await new Promise(() => undefined); } - return { result: { value: "recovered" } }; + return method === "Runtime.evaluate" ? { result: { value: "recovered" } } : undefined; }); fromId.mockReturnValue({ id: 43, diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 2e64d86a0f7..aec4e115024 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -895,7 +895,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }; yield* pushAction(tabId, actionEvent); const epoch = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0; - const control = yield* ensureControlSession(wc); const execute = Effect.fn("PreviewManager.executeControlAction")(function* () { yield* update(tabId, { controller: "agent" }); const send: SendCommand = Effect.fn("PreviewManager.sendCommand")( @@ -938,6 +937,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); }, ); + const colorScheme = (yield* SynchronizedRef.get(tabsRef)).get(tabId)?.colorScheme ?? "system"; + if (colorScheme !== "system") { + yield* send("Emulation.setEmulatedMedia", { + features: [{ name: "prefers-color-scheme", value: colorScheme }], + }); + } return yield* use(send, sendCleanup); }); const finalize = Effect.fn("PreviewManager.finalizeControlAction")(function* ( @@ -973,14 +978,19 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (tabs.has(tabId)) yield* update(tabId, { controller: "none" }); }); const boundedExecution = Effect.gen(function* () { - const result = yield* control.semaphore - .withPermit(execute()) - .pipe(Effect.timeoutOption(Math.max(1, timeoutMs - AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS))); - if (Option.isNone(result)) { - return yield* new PreviewAutomationTimeoutError({ tabId, timeoutMs }); - } - return result.value; - }); + // Session initialization itself sends CDP commands. Keep it inside the + // operation deadline so an offscreen or suspended guest cannot retain + // the synchronized session lock indefinitely and poison later actions. + const control = yield* ensureControlSession(wc); + return yield* control.semaphore.withPermit(execute()); + }).pipe( + Effect.timeoutOption(Math.max(1, timeoutMs - AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS)), + Effect.flatMap((result) => + Option.isNone(result) + ? Effect.fail(new PreviewAutomationTimeoutError({ tabId, timeoutMs })) + : Effect.succeed(result.value), + ), + ); return yield* boundedExecution.pipe( Effect.onExit(finalize), Effect.tapError((error) => @@ -1410,7 +1420,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc.getZoomFactor(), ); yield* attachListeners(tabId, wc); - runFork(restoreControlSession(tabId, wc)); + // The default scheme needs no CDP state, so keep debugger attachment lazy + // for the common offscreen-registration path. A persisted override still + // needs to follow a replaced guest immediately; its restore path is + // separately bounded and releases a stalled session. + if (tab.colorScheme !== "system") runFork(restoreControlSession(tabId, wc)); const registeredAt = yield* currentIso; const registration = yield* SynchronizedRef.modify(tabsRef, (tabs) => { const current = tabs.get(tabId); @@ -1549,9 +1563,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } yield* detachControlSession(wc.id); yield* attempt({ operation: "openDevTools", tabId, webContentsId: wc.id }, () => { - wc.once("devtools-closed", () => { - if (!wc.isDestroyed()) runFork(restoreControlSession(tabId, wc)); - }); wc.openDevTools({ mode: "detach" }); }); }); @@ -1736,12 +1747,18 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function // session attaches so a concurrent setColorScheme is not overwritten with // a stale snapshot. const restoreControlSession = (tabId: string, wc: Electron.WebContents) => - ensureControlSession(wc).pipe( - Effect.andThen(SynchronizedRef.get(tabsRef)), - Effect.flatMap((tabs) => { - const colorScheme = tabs.get(tabId)?.colorScheme ?? "system"; - return colorScheme === "system" ? Effect.void : applyColorScheme(tabId, wc, colorScheme); - }), + Effect.gen(function* () { + yield* ensureControlSession(wc); + const tabs = yield* SynchronizedRef.get(tabsRef); + const colorScheme = tabs.get(tabId)?.colorScheme ?? "system"; + if (colorScheme !== "system") yield* applyColorScheme(tabId, wc, colorScheme); + }).pipe( + Effect.timeoutOption( + Math.max(1, DEFAULT_AUTOMATION_TIMEOUT_MS - AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS), + ), + Effect.flatMap((result) => + Option.isSome(result) ? Effect.void : detachControlSession(wc.id), + ), Effect.ignore, ); diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index b5342b55caa..7a9b13f06a2 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -54,7 +54,7 @@ export const PreviewStatusTool = Tool.make("preview_status", { export const PreviewOpenTool = browserTool( Tool.make("preview_open", { description: - "Show and initialize a collaborative browser tab. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab. Newly created tabs return after creation; wait on the returned tab before interacting while its initial page loads.", + "Show and initialize a collaborative browser tab. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab. Newly created tabs return after server creation while requested presentation and page loading continue; reopening an existing shown tab waits for stable panel presentation. Wait on the returned tab before interacting while its initial page loads.", parameters: PreviewAutomationOpenInput, success: PreviewAutomationStatus, failure: PreviewAutomationError, diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index b88375fd249..1a499f13f13 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -9,7 +9,11 @@ import { usePreviewBridge } from "~/components/preview/usePreviewBridge"; import { cn } from "~/lib/utils"; import { stopBrowserRecording, useActiveBrowserRecordingTabId } from "./browserRecording"; -import { resolveBrowserSurfacePanelRect, useBrowserSurfaceStore } from "./browserSurfaceStore"; +import { + resolveBrowserSurfaceBackgroundCaptureRect, + resolveBrowserSurfacePanelRect, + useBrowserSurfaceStore, +} from "./browserSurfaceStore"; import { browserViewportSettingKey } from "./browserViewportLayout"; import { BrowserDeviceToolbar } from "./BrowserDeviceToolbar"; import { BrowserViewportResizeHandles } from "./BrowserViewportResizeHandles"; @@ -51,9 +55,18 @@ export function HostedBrowserWebview(props: { const presentation = useBrowserSurfaceStore( useShallow((state) => { const current = state.byTabId[tabId]; + const backgroundCapture = (state.backgroundCaptureCountByTabId[tabId] ?? 0) > 0; + const panelRect = resolveBrowserSurfacePanelRect(state.byTabId, tabId); return { - backgroundCapture: (state.backgroundCaptureCountByTabId[tabId] ?? 0) > 0, - rect: resolveBrowserSurfacePanelRect(state.byTabId, tabId), + backgroundCapture, + rect: + panelRect ?? + (backgroundCapture + ? resolveBrowserSurfaceBackgroundCaptureRect(state.byTabId, tabId, { + width: window.innerWidth, + height: window.innerHeight, + }) + : null), visible: current?.visible ?? false, }; }), diff --git a/apps/web/src/browser/browserSurfaceStore.test.ts b/apps/web/src/browser/browserSurfaceStore.test.ts index 992c52cc599..ca753a5821e 100644 --- a/apps/web/src/browser/browserSurfaceStore.test.ts +++ b/apps/web/src/browser/browserSurfaceStore.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it } from "vite-plus/test"; import { acquireBrowserSurface, acquireBrowserSurfaceBackgroundCapture, + resolveBrowserSurfaceBackgroundCaptureRect, resolveBrowserSurfacePanelRect, useBrowserSurfaceStore, } from "./browserSurfaceStore"; @@ -65,6 +66,34 @@ describe("browserSurfaceStore", () => { ).toEqual(liveRect); }); + it("stages a never-presented background tab inside the renderer viewport", () => { + expect( + resolveBrowserSurfaceBackgroundCaptureRect({}, "never-presented", { + width: 1440, + height: 900, + }), + ).toEqual({ + x: 80, + y: 50, + width: 1280, + height: 800, + }); + }); + + it("fits background capture staging to a smaller renderer viewport", () => { + expect( + resolveBrowserSurfaceBackgroundCaptureRect({}, "never-presented", { + width: 800, + height: 600, + }), + ).toEqual({ + x: 0, + y: 0, + width: 800, + height: 600, + }); + }); + it("ignores updates and releases from a stale surface lease", () => { const tabId = "leased-browser-surface"; const staleRect = { x: 0, y: 0, width: 500, height: 700 }; diff --git a/apps/web/src/browser/browserSurfaceStore.ts b/apps/web/src/browser/browserSurfaceStore.ts index 5ce4d9e78d3..6f386d48e01 100644 --- a/apps/web/src/browser/browserSurfaceStore.ts +++ b/apps/web/src/browser/browserSurfaceStore.ts @@ -66,6 +66,28 @@ export function resolveBrowserSurfacePanelRect( return latestVisible?.rect ?? current?.rect ?? null; } +export function resolveBrowserSurfaceBackgroundCaptureRect( + byTabId: Readonly>, + tabId: string, + viewport: { readonly width: number; readonly height: number }, +): BrowserSurfaceRect { + const presentedRect = resolveBrowserSurfacePanelRect(byTabId, tabId); + if (presentedRect) return presentedRect; + + const viewportWidth = + Number.isFinite(viewport.width) && viewport.width > 0 ? Math.round(viewport.width) : 1280; + const viewportHeight = + Number.isFinite(viewport.height) && viewport.height > 0 ? Math.round(viewport.height) : 800; + const width = Math.max(1, Math.min(1280, viewportWidth)); + const height = Math.max(1, Math.min(800, viewportHeight)); + return { + x: Math.max(0, Math.round((viewportWidth - width) / 2)), + y: Math.max(0, Math.round((viewportHeight - height) / 2)), + width, + height, + }; +} + const rectEquals = (left: BrowserSurfaceRect | null, right: BrowserSurfaceRect): boolean => left !== null && left.x === right.x && diff --git a/apps/web/src/components/auth/PairingRouteSurface.logic.test.ts b/apps/web/src/components/auth/PairingRouteSurface.logic.test.ts new file mode 100644 index 00000000000..4c8657aff62 --- /dev/null +++ b/apps/web/src/components/auth/PairingRouteSurface.logic.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { claimPairingToken } from "./PairingRouteSurface.logic"; + +describe("claimPairingToken", () => { + it("claims each pairing token once", () => { + const attemptedTokens = new Set(); + + expect(claimPairingToken("first-token", attemptedTokens)).toBe("first-token"); + expect(claimPairingToken("first-token", attemptedTokens)).toBeNull(); + expect(claimPairingToken("second-token", attemptedTokens)).toBe("second-token"); + }); + + it("ignores a URL without a pairing token", () => { + expect(claimPairingToken(null, new Set())).toBeNull(); + }); +}); diff --git a/apps/web/src/components/auth/PairingRouteSurface.logic.ts b/apps/web/src/components/auth/PairingRouteSurface.logic.ts new file mode 100644 index 00000000000..6b1e729a592 --- /dev/null +++ b/apps/web/src/components/auth/PairingRouteSurface.logic.ts @@ -0,0 +1,8 @@ +export function claimPairingToken( + token: string | null, + attemptedTokens: Set, +): string | null { + if (!token || attemptedTokens.has(token)) return null; + attemptedTokens.add(token); + return token; +} diff --git a/apps/web/src/components/auth/PairingRouteSurface.tsx b/apps/web/src/components/auth/PairingRouteSurface.tsx index 59288506569..b066fdede47 100644 --- a/apps/web/src/components/auth/PairingRouteSurface.tsx +++ b/apps/web/src/components/auth/PairingRouteSurface.tsx @@ -13,6 +13,7 @@ import { readHostedPairingRequest } from "../../hostedPairing"; import { Button } from "../ui/button"; import { Input } from "../ui/input"; import { useAtomCommand } from "../../state/use-atom-command"; +import { claimPairingToken } from "./PairingRouteSurface.logic"; export function PairingPendingSurface() { return ( @@ -47,11 +48,10 @@ export function PairingRouteSurface({ initialErrorMessage?: string; onAuthenticated: () => void; }) { - const autoPairTokenRef = useRef(peekPairingTokenFromUrl()); - const [credential, setCredential] = useState(() => autoPairTokenRef.current ?? ""); + const [credential, setCredential] = useState(() => peekPairingTokenFromUrl() ?? ""); const [errorMessage, setErrorMessage] = useState(initialErrorMessage ?? ""); const [isSubmitting, setIsSubmitting] = useState(false); - const autoSubmitAttemptedRef = useRef(false); + const attemptedPairingTokensRef = useRef(new Set()); const submitCredential = useCallback( async (nextCredential: string) => { @@ -86,14 +86,20 @@ export function PairingRouteSurface({ ); useEffect(() => { - const token = autoPairTokenRef.current; - if (!token || autoSubmitAttemptedRef.current) { - return; - } - - autoSubmitAttemptedRef.current = true; - stripPairingTokenFromUrl(); - void submitCredential(token); + const submitPairingTokenFromUrl = () => { + const token = claimPairingToken(peekPairingTokenFromUrl(), attemptedPairingTokensRef.current); + if (!token) return; + + setCredential(token); + stripPairingTokenFromUrl(); + void submitCredential(token); + }; + + submitPairingTokenFromUrl(); + window.addEventListener("hashchange", submitPairingTokenFromUrl); + return () => { + window.removeEventListener("hashchange", submitPairingTokenFromUrl); + }; }, [submitCredential]); return ( diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 1322f9ffbd0..12c4f8d66a4 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -29,7 +29,7 @@ import { reconcilePreviewServerSessions, updatePreviewServerSnapshot, } from "~/previewStateStore"; -import { useRightPanelStore } from "~/rightPanelStore"; +import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelStore"; import { resolveBrowserNavigationTarget } from "~/browser/browserTargetResolver"; import { readActiveBrowserRecordingTabId, @@ -48,6 +48,10 @@ import { useAtomQueryRunner } from "~/state/use-atom-query-runner"; import { useAtomCommand } from "~/state/use-atom-command"; import { previewBridge } from "./previewBridge"; +import { + isPreviewAutomationTabPresented, + revealPreviewAutomationTab, +} from "./previewAutomationPresentation"; import { PreviewAutomationNavigationTimeoutError, PreviewAutomationOperationError, @@ -67,6 +71,8 @@ import { } from "./previewAutomationTarget"; import { isPreviewViewportReady } from "./previewViewportReadiness"; +const PREVIEW_AUTOMATION_INTERNAL_RESPONSE_GRACE_MS = 500; + const waitForDesktopOverlay = async ( threadRef: ScopedThreadRef, requestId: string, @@ -94,20 +100,38 @@ const waitForBrowserSurfaceVisibility = async ( threadRef: ScopedThreadRef, requestId: string, tabId: string, - requestTimeoutMs: number, + timeoutMs: number, ): Promise => { - const timeoutMs = Math.max(1, Math.min(2_000, requestTimeoutMs - 250)); const deadline = Date.now() + timeoutMs; + let presentedSince: number | null = null; while (Date.now() <= deadline) { - if (useBrowserSurfaceStore.getState().byTabId[tabId]?.visible) return; + const now = Date.now(); + if (isPreviewAutomationTabPresented(threadRef, tabId)) { + presentedSince ??= now; + // Require the selection to survive multiple presentation updates. A + // single transient `visible` frame can otherwise make open acknowledge + // just before routing or panel reconciliation unmounts the surface. + if (now - presentedSince >= 100) return; + } else { + presentedSince = null; + // Session reconciliation and route hydration can race a cold open. + // Reassert the explicit show request only while that request is pending. + revealPreviewAutomationTab(threadRef, tabId); + } await new Promise((resolve) => window.setTimeout(resolve, 50)); } + const panel = selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, threadRef); + const presentation = useBrowserSurfaceStore.getState().byTabId[tabId]; throw new PreviewAutomationVisibilityTimeoutError({ requestId, environmentId: threadRef.environmentId, threadId: threadRef.threadId, tabId, timeoutMs, + activeSurfaceId: panel.activeSurfaceId, + rightPanelOpen: panel.isOpen, + surfaceRegistered: panel.surfaces.some((surface) => surface.id === `browser:${tabId}`), + presentationRectAvailable: presentation?.rect !== null && presentation?.rect !== undefined, }); }; @@ -133,6 +157,22 @@ const waitForBackgroundCapturePresentation = async (tabId: string): Promise( + tabId: string, + use: (background: boolean) => Promise, +): Promise => { + const background = !(useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false); + if (!background) return await use(false); + + const releaseCapture = acquireBrowserSurfaceBackgroundCapture(tabId); + try { + await waitForBackgroundCapturePresentation(tabId); + return await use(true); + } finally { + releaseCapture(); + } +}; + const waitForNavigationReadiness = async ( threadRef: ScopedThreadRef, requestId: string, @@ -348,6 +388,10 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) const handleRequest = useCallback( async (request: PreviewAutomationRequest): Promise => { + const operationDeadline = + Date.now() + Math.max(1, request.timeoutMs - PREVIEW_AUTOMATION_INTERNAL_RESPONSE_GRACE_MS); + const remainingOperationBudget = (requestedTimeoutMs = request.timeoutMs): number => + Math.max(1, Math.min(requestedTimeoutMs, operationDeadline - Date.now())); const threadRef: ScopedThreadRef = { environmentId, threadId: request.threadId, @@ -384,7 +428,12 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) if (!bridge || !readyTabId) { throw new PreviewAutomationTargetUnavailableError(unavailableTarget); } - await waitForDesktopOverlay(threadRef, request.requestId, readyTabId, request.timeoutMs); + await waitForDesktopOverlay( + threadRef, + request.requestId, + readyTabId, + remainingOperationBudget(), + ); return { bridge, tabId: readyTabId }; }; switch (request.operation) { @@ -426,7 +475,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) tabId = activeTabId; } if (input.show ?? true) { - useRightPanelStore.getState().openBrowser(threadRef, activeTabId); + revealPreviewAutomationTab(threadRef, activeTabId); } const waitPolicy = activeSnapshot ? resolvePreviewAutomationOpenWaitPolicy(input, activeSnapshot, reusedExistingTab) @@ -439,7 +488,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) threadRef, request.requestId, activeTabId, - request.timeoutMs, + remainingOperationBudget(), ); } if (reusedExistingTab && resolvedInputUrl && previewBridge) { @@ -449,7 +498,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) request.requestId, activeTabId, "load", - request.timeoutMs, + remainingOperationBudget(), ); } if (waitPolicy?.waitForVisibility) { @@ -457,7 +506,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) threadRef, request.requestId, activeTabId, - request.timeoutMs, + remainingOperationBudget(), ); } return await currentStatus(threadRef, activeTabId); @@ -473,12 +522,16 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) }, ); await ready.bridge.navigate(ready.tabId, resolution.resolvedUrl); - await waitForNavigationReadiness( - threadRef, - request.requestId, + await withBackgroundAutomationPresentation( ready.tabId, - input.readiness ?? "load", - input.timeoutMs ?? request.timeoutMs, + async () => + await waitForNavigationReadiness( + threadRef, + request.requestId, + ready.tabId, + input.readiness ?? "load", + remainingOperationBudget(input.timeoutMs ?? request.timeoutMs), + ), ); return await currentStatus(threadRef, ready.tabId); } @@ -501,7 +554,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) const viewport = await waitForRenderedViewport( ready.tabId, setting, - input.timeoutMs ?? request.timeoutMs, + remainingOperationBudget(input.timeoutMs ?? request.timeoutMs), { requestId: request.requestId, environmentId, @@ -517,7 +570,10 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) case "setColorScheme": { const ready = await requireReadyTab(); const input = request.input as PreviewAutomationSetColorSchemeInput; - await ready.bridge.setColorScheme(ready.tabId, input.colorScheme); + await withBackgroundAutomationPresentation( + ready.tabId, + async () => await ready.bridge.setColorScheme(ready.tabId, input.colorScheme), + ); return { tabId: ready.tabId, colorScheme: input.colorScheme, @@ -525,60 +581,75 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "snapshot": { const ready = await requireReadyTab(); - const background = !( - useBrowserSurfaceStore.getState().byTabId[ready.tabId]?.visible ?? false + return await withBackgroundAutomationPresentation( + ready.tabId, + async (background) => await ready.bridge.automation.snapshot(ready.tabId, background), ); - if (!background) { - return await ready.bridge.automation.snapshot(ready.tabId, false); - } - const releaseCapture = acquireBrowserSurfaceBackgroundCapture(ready.tabId); - try { - await waitForBackgroundCapturePresentation(ready.tabId); - return await ready.bridge.automation.snapshot(ready.tabId, true); - } finally { - releaseCapture(); - } } case "click": { const ready = await requireReadyTab(); - return await ready.bridge.automation.click( + return await withBackgroundAutomationPresentation( ready.tabId, - request.input as Parameters[1], + async () => + await ready.bridge.automation.click( + ready.tabId, + request.input as Parameters[1], + ), ); } case "type": { const ready = await requireReadyTab(); - return await ready.bridge.automation.type( + return await withBackgroundAutomationPresentation( ready.tabId, - request.input as Parameters[1], + async () => + await ready.bridge.automation.type( + ready.tabId, + request.input as Parameters[1], + ), ); } case "press": { const ready = await requireReadyTab(); - return await ready.bridge.automation.press( + return await withBackgroundAutomationPresentation( ready.tabId, - request.input as Parameters[1], + async () => + await ready.bridge.automation.press( + ready.tabId, + request.input as Parameters[1], + ), ); } case "scroll": { const ready = await requireReadyTab(); - return await ready.bridge.automation.scroll( + return await withBackgroundAutomationPresentation( ready.tabId, - request.input as Parameters[1], + async () => + await ready.bridge.automation.scroll( + ready.tabId, + request.input as Parameters[1], + ), ); } case "evaluate": { const ready = await requireReadyTab(); - return await ready.bridge.automation.evaluate( + return await withBackgroundAutomationPresentation( ready.tabId, - request.input as Parameters[1], + async () => + await ready.bridge.automation.evaluate( + ready.tabId, + request.input as Parameters[1], + ), ); } case "waitFor": { const ready = await requireReadyTab(); - return await ready.bridge.automation.waitFor( + return await withBackgroundAutomationPresentation( ready.tabId, - request.input as Parameters[1], + async () => + await ready.bridge.automation.waitFor( + ready.tabId, + request.input as Parameters[1], + ), ); } case "recordingStart": { diff --git a/apps/web/src/components/preview/previewAutomationErrors.ts b/apps/web/src/components/preview/previewAutomationErrors.ts index 63b8767ac75..cb07b3ac38c 100644 --- a/apps/web/src/components/preview/previewAutomationErrors.ts +++ b/apps/web/src/components/preview/previewAutomationErrors.ts @@ -83,6 +83,10 @@ export class PreviewAutomationVisibilityTimeoutError extends Schema.TaggedErrorC threadId: ThreadId, tabId: PreviewTabId, timeoutMs: Schema.Int, + activeSurfaceId: Schema.optional(Schema.NullOr(Schema.String)), + rightPanelOpen: Schema.optional(Schema.Boolean), + surfaceRegistered: Schema.optional(Schema.Boolean), + presentationRectAvailable: Schema.optional(Schema.Boolean), }, ) { get responseTag() { diff --git a/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts b/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts index 9d959a2efac..0c00954eb37 100644 --- a/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts +++ b/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts @@ -13,7 +13,7 @@ const snapshot = (navStatus: PreviewSessionSnapshot["navStatus"]): PreviewSessio }); describe("preview automation open readiness", () => { - it("acknowledges a newly created URL tab before cold renderer readiness", () => { + it("acknowledges a newly created shown tab without cold renderer readiness", () => { expect( resolvePreviewAutomationOpenWaitPolicy( { url: "https://example.com" } as PreviewAutomationOpenInput, @@ -31,6 +31,24 @@ describe("preview automation open readiness", () => { }); }); + it("acknowledges a newly created background tab without renderer readiness", () => { + expect( + resolvePreviewAutomationOpenWaitPolicy( + { url: "https://example.com", show: false } as PreviewAutomationOpenInput, + snapshot({ + _tag: "Loading", + url: "https://example.com/", + title: "", + }), + false, + ), + ).toEqual({ + acknowledgeAfterCreation: true, + waitForOverlay: false, + waitForVisibility: false, + }); + }); + it("waits for the overlay and visibility when navigating a reused tab", () => { expect( resolvePreviewAutomationOpenWaitPolicy( diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts new file mode 100644 index 00000000000..c463a6537f8 --- /dev/null +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -0,0 +1,60 @@ +import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { EnvironmentId, type PreviewSessionSnapshot, ThreadId } from "@t3tools/contracts"; +import { beforeEach, describe, expect, it } from "vite-plus/test"; + +import { acquireBrowserSurface, useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; +import { + applyPreviewServerSnapshot, + readThreadPreviewState, + resetPreviewStateForTests, +} from "~/previewStateStore"; +import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelStore"; + +import { + isPreviewAutomationTabPresented, + revealPreviewAutomationTab, +} from "./previewAutomationPresentation"; + +const threadRef = scopeThreadRef(EnvironmentId.make("environment-1"), ThreadId.make("thread-1")); + +const snapshot = (tabId: string, updatedAt: string): PreviewSessionSnapshot => ({ + threadId: threadRef.threadId, + tabId, + navStatus: { _tag: "Idle" }, + canGoBack: false, + canGoForward: false, + updatedAt, +}); + +describe("preview automation presentation", () => { + beforeEach(() => { + resetPreviewStateForTests(); + useRightPanelStore.setState({ byThreadKey: {} }); + useBrowserSurfaceStore.setState({ byTabId: {}, backgroundCaptureCountByTabId: {} }); + }); + + it("selects the requested preview tab and its right-panel surface together", () => { + applyPreviewServerSnapshot(threadRef, snapshot("tab-1", "2026-07-25T00:00:00.000Z")); + applyPreviewServerSnapshot(threadRef, snapshot("tab-2", "2026-07-25T00:00:01.000Z")); + + revealPreviewAutomationTab(threadRef, "tab-1"); + + expect(readThreadPreviewState(threadRef).activeTabId).toBe("tab-1"); + expect( + selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, threadRef), + ).toMatchObject({ + isOpen: true, + activeSurfaceId: "browser:tab-1", + }); + expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(false); + + const surface = acquireBrowserSurface("tab-1"); + surface.present({ x: 0, y: 0, width: 800, height: 600 }, true); + + expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(true); + + useRightPanelStore.getState().openBrowser(threadRef, "tab-2"); + expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(false); + surface.release(); + }); +}); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts new file mode 100644 index 00000000000..e881f73b79a --- /dev/null +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -0,0 +1,19 @@ +import type { ScopedThreadRef } from "@t3tools/contracts"; + +import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; +import { setActivePreviewTab } from "~/previewStateStore"; +import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelStore"; + +export function revealPreviewAutomationTab(ref: ScopedThreadRef, tabId: string): void { + setActivePreviewTab(ref, tabId); + useRightPanelStore.getState().openBrowser(ref, tabId); +} + +export function isPreviewAutomationTabPresented(ref: ScopedThreadRef, tabId: string): boolean { + const panel = selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, ref); + return ( + panel.isOpen && + panel.activeSurfaceId === `browser:${tabId}` && + (useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false) + ); +} diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts index 1a86a60201f..8f9a0014ec4 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts @@ -300,6 +300,10 @@ describe("previewAutomationRequestConsumer", () => { threadId, tabId, timeoutMs: 2_000, + activeSurfaceId: "browser:tab-1", + rightPanelOpen: true, + surfaceRegistered: true, + presentationRectAvailable: false, }); expect( @@ -312,7 +316,14 @@ describe("previewAutomationRequestConsumer", () => { }), ).toMatchObject({ _tag: "PreviewAutomationTimeoutError", - detail: { tabId: "tab-1", timeoutMs: 2_000 }, + detail: { + tabId: "tab-1", + timeoutMs: 2_000, + activeSurfaceId: "browser:tab-1", + rightPanelOpen: true, + surfaceRegistered: true, + presentationRectAvailable: false, + }, }); }); diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 9460ed78287..0f2bde55cd9 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -107,7 +107,7 @@ export const PreviewAutomationOpenInput = Schema.Struct({ ) .annotate({ description: - "Opens the collaborative browser for the current thread. Newly created tabs acknowledge creation before renderer readiness; wait on the returned tab before interacting while its initial page loads.", + "Opens the collaborative browser for the current thread. Newly created tabs acknowledge server creation immediately while any requested presentation and initial page load continue; reopening an existing shown tab waits for stable panel presentation. Wait on the returned tab before interacting while its initial page loads.", }); export type PreviewAutomationOpenInput = typeof PreviewAutomationOpenInput.Type; diff --git a/scripts/dev-runner.test.ts b/scripts/dev-runner.test.ts index 3b79db49f5b..cf8d55c763b 100644 --- a/scripts/dev-runner.test.ts +++ b/scripts/dev-runner.test.ts @@ -206,8 +206,8 @@ it.layer(NodeServices.layer)("dev-runner", (it) => { assert.equal(env.T3CODE_HOME, path.resolve("/tmp/custom-t3")); assert.equal(env.T3CODE_PORT, "4222"); - assert.equal(env.VITE_HTTP_URL, "http://localhost:4222"); - assert.equal(env.VITE_WS_URL, "ws://localhost:4222"); + assert.equal(env.VITE_HTTP_URL, "http://127.0.0.1:4222"); + assert.equal(env.VITE_WS_URL, "ws://127.0.0.1:4222"); assert.equal(env.T3CODE_NO_BROWSER, "1"); assert.equal(env.T3CODE_AUTO_BOOTSTRAP_PROJECT_FROM_CWD, "0"); assert.equal(env.T3CODE_LOG_WS_EVENTS, "1"); @@ -336,8 +336,10 @@ it.layer(NodeServices.layer)("dev-runner", (it) => { }); assert.equal(env.T3CODE_PORT, "13773"); - assert.equal(env.VITE_HTTP_URL, "http://localhost:13773"); - assert.equal(env.VITE_WS_URL, "ws://localhost:13773"); + assert.equal(env.VITE_HTTP_URL, "http://127.0.0.1:13773"); + assert.equal(env.VITE_WS_URL, "ws://127.0.0.1:13773"); + assert.equal(env.VITE_DEV_SERVER_URL, "http://127.0.0.1:5733"); + assert.equal(env.HOST, "127.0.0.1"); }), ); }); diff --git a/scripts/dev-runner.ts b/scripts/dev-runner.ts index 1938232300f..b3506140903 100644 --- a/scripts/dev-runner.ts +++ b/scripts/dev-runner.ts @@ -26,7 +26,7 @@ const BASE_SERVER_PORT = 13773; const BASE_WEB_PORT = 5733; const MAX_HASH_OFFSET = 3000; const MAX_PORT = 65535; -const DESKTOP_DEV_LOOPBACK_HOST = "127.0.0.1"; +const DEV_LOOPBACK_HOST = "127.0.0.1"; const DEV_PORT_PROBE_HOSTS = ["127.0.0.1", "0.0.0.0", "::1", "::"] as const; export const DEFAULT_T3_HOME = Effect.map(Effect.service(Path.Path), (path) => @@ -252,9 +252,7 @@ export function createDevRunnerEnv({ const output: NodeJS.ProcessEnv = { ...baseEnv, PORT: String(webPort), - VITE_DEV_SERVER_URL: - devUrl?.toString() ?? - `http://${isDesktopMode ? DESKTOP_DEV_LOOPBACK_HOST : "localhost"}:${webPort}`, + VITE_DEV_SERVER_URL: devUrl?.toString() ?? `http://${DEV_LOOPBACK_HOST}:${webPort}`, }; if (configuredBaseDir !== undefined) { @@ -265,12 +263,12 @@ export function createDevRunnerEnv({ if (!isDesktopMode) { output.T3CODE_PORT = String(serverPort); - output.VITE_HTTP_URL = `http://localhost:${serverPort}`; - output.VITE_WS_URL = `ws://localhost:${serverPort}`; + output.VITE_HTTP_URL = `http://${DEV_LOOPBACK_HOST}:${serverPort}`; + output.VITE_WS_URL = `ws://${DEV_LOOPBACK_HOST}:${serverPort}`; } else { output.T3CODE_PORT = String(serverPort); - output.VITE_HTTP_URL = `http://${DESKTOP_DEV_LOOPBACK_HOST}:${serverPort}`; - output.VITE_WS_URL = `ws://${DESKTOP_DEV_LOOPBACK_HOST}:${serverPort}`; + output.VITE_HTTP_URL = `http://${DEV_LOOPBACK_HOST}:${serverPort}`; + output.VITE_WS_URL = `ws://${DEV_LOOPBACK_HOST}:${serverPort}`; delete output.T3CODE_MODE; delete output.T3CODE_NO_BROWSER; delete output.T3CODE_HOST; @@ -306,8 +304,11 @@ export function createDevRunnerEnv({ delete output.T3CODE_DESKTOP_WS_URL; } + if (mode !== "dev:server") { + output.HOST = DEV_LOOPBACK_HOST; + } + if (isDesktopMode) { - output.HOST = DESKTOP_DEV_LOOPBACK_HOST; delete output.T3CODE_DESKTOP_WS_URL; } From 76c8bd61d965749119b73b1371fc745352d181ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 25 Jul 2026 22:12:52 +0100 Subject: [PATCH 05/43] Bound preview automation staging deadlines Report background presentation deadlines as typed timeouts. --- .../preview/PreviewAutomationHosts.tsx | 60 ++++++++++++------- .../preview/previewAutomationErrors.ts | 20 +++++++ .../previewAutomationPresentation.test.ts | 33 +++++++++- .../preview/previewAutomationPresentation.ts | 40 +++++++++++++ .../previewAutomationRequestConsumer.test.ts | 24 ++++++++ 5 files changed, 153 insertions(+), 24 deletions(-) diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 12c4f8d66a4..6a4330302ac 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -51,6 +51,7 @@ import { previewBridge } from "./previewBridge"; import { isPreviewAutomationTabPresented, revealPreviewAutomationTab, + waitForPreviewAutomationBackgroundPresentation, } from "./previewAutomationPresentation"; import { PreviewAutomationNavigationTimeoutError, @@ -135,30 +136,11 @@ const waitForBrowserSurfaceVisibility = async ( }); }; -const waitForBackgroundCapturePresentation = async (tabId: string): Promise => { - const deadline = Date.now() + 1_000; - while (Date.now() <= deadline) { - const wrapper = Array.from( - document.querySelectorAll("[data-preview-viewport]"), - ).find( - (candidate) => - candidate.dataset["previewViewport"] === tabId && - candidate.dataset["previewBackgroundCapture"] === "true", - ); - if (wrapper) { - // Force the staged wrapper through layout, then allow Chromium two - // compositor frames before asking the guest WebContents for pixels. - void wrapper.offsetWidth; - await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); - await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); - return; - } - await new Promise((resolve) => window.setTimeout(resolve, 16)); - } -}; - const withBackgroundAutomationPresentation = async ( + threadRef: ScopedThreadRef, + requestId: string, tabId: string, + timeoutMs: number, use: (background: boolean) => Promise, ): Promise => { const background = !(useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false); @@ -166,7 +148,12 @@ const withBackgroundAutomationPresentation = async ( const releaseCapture = acquireBrowserSurfaceBackgroundCapture(tabId); try { - await waitForBackgroundCapturePresentation(tabId); + await waitForPreviewAutomationBackgroundPresentation({ + threadRef, + requestId, + tabId, + timeoutMs, + }); return await use(true); } finally { releaseCapture(); @@ -523,7 +510,10 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) ); await ready.bridge.navigate(ready.tabId, resolution.resolvedUrl); await withBackgroundAutomationPresentation( + threadRef, + request.requestId, ready.tabId, + remainingOperationBudget(), async () => await waitForNavigationReadiness( threadRef, @@ -571,7 +561,10 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) const ready = await requireReadyTab(); const input = request.input as PreviewAutomationSetColorSchemeInput; await withBackgroundAutomationPresentation( + threadRef, + request.requestId, ready.tabId, + remainingOperationBudget(), async () => await ready.bridge.setColorScheme(ready.tabId, input.colorScheme), ); return { @@ -582,14 +575,20 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) case "snapshot": { const ready = await requireReadyTab(); return await withBackgroundAutomationPresentation( + threadRef, + request.requestId, ready.tabId, + remainingOperationBudget(), async (background) => await ready.bridge.automation.snapshot(ready.tabId, background), ); } case "click": { const ready = await requireReadyTab(); return await withBackgroundAutomationPresentation( + threadRef, + request.requestId, ready.tabId, + remainingOperationBudget(), async () => await ready.bridge.automation.click( ready.tabId, @@ -600,7 +599,10 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) case "type": { const ready = await requireReadyTab(); return await withBackgroundAutomationPresentation( + threadRef, + request.requestId, ready.tabId, + remainingOperationBudget(), async () => await ready.bridge.automation.type( ready.tabId, @@ -611,7 +613,10 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) case "press": { const ready = await requireReadyTab(); return await withBackgroundAutomationPresentation( + threadRef, + request.requestId, ready.tabId, + remainingOperationBudget(), async () => await ready.bridge.automation.press( ready.tabId, @@ -622,7 +627,10 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) case "scroll": { const ready = await requireReadyTab(); return await withBackgroundAutomationPresentation( + threadRef, + request.requestId, ready.tabId, + remainingOperationBudget(), async () => await ready.bridge.automation.scroll( ready.tabId, @@ -633,7 +641,10 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) case "evaluate": { const ready = await requireReadyTab(); return await withBackgroundAutomationPresentation( + threadRef, + request.requestId, ready.tabId, + remainingOperationBudget(), async () => await ready.bridge.automation.evaluate( ready.tabId, @@ -644,7 +655,10 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) case "waitFor": { const ready = await requireReadyTab(); return await withBackgroundAutomationPresentation( + threadRef, + request.requestId, ready.tabId, + remainingOperationBudget(), async () => await ready.bridge.automation.waitFor( ready.tabId, diff --git a/apps/web/src/components/preview/previewAutomationErrors.ts b/apps/web/src/components/preview/previewAutomationErrors.ts index cb07b3ac38c..68fc90203f1 100644 --- a/apps/web/src/components/preview/previewAutomationErrors.ts +++ b/apps/web/src/components/preview/previewAutomationErrors.ts @@ -98,6 +98,25 @@ export class PreviewAutomationVisibilityTimeoutError extends Schema.TaggedErrorC } } +export class PreviewAutomationBackgroundPresentationTimeoutError extends Schema.TaggedErrorClass()( + "PreviewAutomationBackgroundPresentationTimeoutError", + { + requestId: TrimmedNonEmptyString, + environmentId: EnvironmentId, + threadId: ThreadId, + tabId: PreviewTabId, + timeoutMs: Schema.Int, + }, +) { + get responseTag() { + return "PreviewAutomationTimeoutError" as const; + } + + override get message(): string { + return `Preview browser surface for request ${this.requestId} on environment ${this.environmentId} thread ${this.threadId} tab ${this.tabId} was not staged for background automation within ${this.timeoutMs}ms.`; + } +} + export class PreviewAutomationHostDeadlineExceededError extends Schema.TaggedErrorClass()( "PreviewAutomationHostDeadlineExceededError", { @@ -253,6 +272,7 @@ export const PreviewAutomationHostError = Schema.Union([ PreviewAutomationNavigationTimeoutError, PreviewAutomationViewportTimeoutError, PreviewAutomationVisibilityTimeoutError, + PreviewAutomationBackgroundPresentationTimeoutError, PreviewAutomationHostDeadlineExceededError, PreviewAutomationTargetUnavailableError, PreviewAutomationRecordingNotActiveError, diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index c463a6537f8..8042ef44e31 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -1,6 +1,6 @@ import { scopeThreadRef } from "@t3tools/client-runtime/environment"; import { EnvironmentId, type PreviewSessionSnapshot, ThreadId } from "@t3tools/contracts"; -import { beforeEach, describe, expect, it } from "vite-plus/test"; +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; import { acquireBrowserSurface, useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; import { @@ -13,6 +13,7 @@ import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelSto import { isPreviewAutomationTabPresented, revealPreviewAutomationTab, + waitForPreviewAutomationBackgroundPresentation, } from "./previewAutomationPresentation"; const threadRef = scopeThreadRef(EnvironmentId.make("environment-1"), ThreadId.make("thread-1")); @@ -57,4 +58,34 @@ describe("preview automation presentation", () => { expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(false); surface.release(); }); + + it("uses the operation budget when background staging does not render", async () => { + vi.useFakeTimers(); + vi.stubGlobal("document", { + querySelectorAll: () => [], + }); + vi.stubGlobal("window", { + setTimeout, + }); + try { + const presentation = waitForPreviewAutomationBackgroundPresentation({ + threadRef, + requestId: "request-background", + tabId: "tab-background", + timeoutMs: 40, + }); + const rejection = expect(presentation).rejects.toMatchObject({ + _tag: "PreviewAutomationBackgroundPresentationTimeoutError", + requestId: "request-background", + tabId: "tab-background", + timeoutMs: 40, + }); + + await vi.advanceTimersByTimeAsync(40); + await rejection; + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); }); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index e881f73b79a..d111a81e403 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -4,6 +4,8 @@ import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; import { setActivePreviewTab } from "~/previewStateStore"; import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelStore"; +import { PreviewAutomationBackgroundPresentationTimeoutError } from "./previewAutomationErrors"; + export function revealPreviewAutomationTab(ref: ScopedThreadRef, tabId: string): void { setActivePreviewTab(ref, tabId); useRightPanelStore.getState().openBrowser(ref, tabId); @@ -17,3 +19,41 @@ export function isPreviewAutomationTabPresented(ref: ScopedThreadRef, tabId: str (useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false) ); } + +export async function waitForPreviewAutomationBackgroundPresentation(input: { + readonly threadRef: ScopedThreadRef; + readonly requestId: string; + readonly tabId: string; + readonly timeoutMs: number; +}): Promise { + const deadline = Date.now() + input.timeoutMs; + while (true) { + const wrapper = Array.from( + document.querySelectorAll("[data-preview-viewport]"), + ).find( + (candidate) => + candidate.dataset["previewViewport"] === input.tabId && + candidate.dataset["previewBackgroundCapture"] === "true", + ); + if (wrapper) { + // Force the staged wrapper through layout, then allow Chromium two + // compositor frames before asking the guest WebContents for pixels. + void wrapper.offsetWidth; + await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); + await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); + return; + } + + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) break; + await new Promise((resolve) => window.setTimeout(resolve, Math.min(16, remainingMs))); + } + + throw new PreviewAutomationBackgroundPresentationTimeoutError({ + requestId: input.requestId, + environmentId: input.threadRef.environmentId, + threadId: input.threadRef.threadId, + tabId: input.tabId, + timeoutMs: input.timeoutMs, + }); +} diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts index 8f9a0014ec4..089a39cf450 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts @@ -10,6 +10,7 @@ import { AsyncResult, Atom, AtomRegistry } from "effect/unstable/reactivity"; import { describe, expect, it, vi } from "vite-plus/test"; import { + PreviewAutomationBackgroundPresentationTimeoutError, PreviewAutomationHostDeadlineExceededError, PreviewAutomationRecordingNotActiveError, PreviewAutomationTargetUnavailableError, @@ -327,6 +328,29 @@ describe("previewAutomationRequestConsumer", () => { }); }); + it("preserves background presentation timeouts as timeout responses", () => { + const error = new PreviewAutomationBackgroundPresentationTimeoutError({ + requestId: "request-background", + environmentId, + threadId, + tabId, + timeoutMs: 1_500, + }); + + expect( + serializePreviewAutomationError(error, { + requestId: "request-background", + operation: "snapshot", + environmentId, + threadId, + tabId, + }), + ).toMatchObject({ + _tag: "PreviewAutomationTimeoutError", + detail: { tabId: "tab-1", timeoutMs: 1_500 }, + }); + }); + it("preserves host response deadline errors as timeout responses", () => { const error = new PreviewAutomationHostDeadlineExceededError({ requestId: "request-snapshot", From c0d9f6756d24419e46dc8867cb13f06c94d345fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 25 Jul 2026 22:37:07 +0100 Subject: [PATCH 06/43] Fix preview automation deadline races Bound background leases and preserve short request timeouts. Isolate queued CDP timeouts and handle hidden empty tabs. --- apps/desktop/src/preview/Manager.test.ts | 72 +++++++++++++++++ apps/desktop/src/preview/Manager.ts | 12 ++- .../preview/PreviewAutomationHosts.tsx | 60 +++++--------- .../previewAutomationOpenReadiness.test.ts | 34 ++++++++ .../preview/previewAutomationOpenReadiness.ts | 6 +- .../previewAutomationPresentation.test.ts | 80 +++++++++++++++++++ .../preview/previewAutomationPresentation.ts | 54 ++++++++++++- .../previewAutomationRequestConsumer.test.ts | 57 +++++++++++++ .../previewAutomationRequestConsumer.ts | 10 ++- 9 files changed, 340 insertions(+), 45 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index b1647188ea5..f087c0b1443 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -366,6 +366,78 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("does not let a queued timeout detach the active control session", () => + withManager((manager) => + Effect.gen(function* () { + let attached = false; + let stallNextEvaluation = true; + const attach = vi.fn(() => { + attached = true; + }); + const detach = vi.fn(() => { + attached = false; + }); + const sendCommand = vi.fn(async (method: string): Promise => { + if (method === "Runtime.evaluate" && stallNextEvaluation) { + stallNextEvaluation = false; + return await new Promise(() => undefined); + } + return method === "Runtime.evaluate" ? { result: { value: "recovered" } } : undefined; + }); + fromId.mockReturnValue({ + id: 44, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => "https://example.com/", + getTitle: () => "Example", + isLoading: () => false, + isDevToolsOpened: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => attached, + attach, + detach, + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + } as never); + + yield* manager.createTab("tab_queued_timeout"); + yield* manager.registerWebview("tab_queued_timeout", 44); + const active = yield* manager + .automationEvaluate("tab_queued_timeout", { expression: "document.title" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + const queued = yield* manager + .automationWaitFor("tab_queued_timeout", { text: "ready", timeoutMs: 1_000 }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + + yield* TestClock.adjust(1_000); + expect(Exit.isFailure(yield* Effect.exit(Fiber.join(queued)))).toBe(true); + expect(detach).not.toHaveBeenCalled(); + + yield* TestClock.adjust(14_000); + expect(Exit.isFailure(yield* Effect.exit(Fiber.join(active)))).toBe(true); + expect(detach).toHaveBeenCalledOnce(); + + expect( + yield* manager.automationEvaluate("tab_queued_timeout", { + expression: "document.title", + }), + ).toBe("recovered"); + }), + ), + ); + effectIt.effect("isolates failed state listeners and continues delivery", () => { const loggedErrors: Array = []; const logger = Logger.make(({ message }) => { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index aec4e115024..53c82c9dd56 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -977,12 +977,18 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const tabs = yield* SynchronizedRef.get(tabsRef); if (tabs.has(tabId)) yield* update(tabId, { controller: "none" }); }); + let detachOnTimeout = true; const boundedExecution = Effect.gen(function* () { // Session initialization itself sends CDP commands. Keep it inside the // operation deadline so an offscreen or suspended guest cannot retain // the synchronized session lock indefinitely and poison later actions. const control = yield* ensureControlSession(wc); - return yield* control.semaphore.withPermit(execute()); + detachOnTimeout = false; + return yield* control.semaphore.withPermit( + Effect.sync(() => { + detachOnTimeout = true; + }).pipe(Effect.andThen(execute())), + ); }).pipe( Effect.timeoutOption(Math.max(1, timeoutMs - AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS)), Effect.flatMap((result) => @@ -994,7 +1000,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return yield* boundedExecution.pipe( Effect.onExit(finalize), Effect.tapError((error) => - isPreviewAutomationTimeoutError(error) ? detachControlSession(wc.id) : Effect.void, + isPreviewAutomationTimeoutError(error) && detachOnTimeout + ? detachControlSession(wc.id) + : Effect.void, ), ); }); diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 6a4330302ac..0b23ea14847 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -37,10 +37,7 @@ import { stopBrowserRecording, } from "~/browser/browserRecording"; import { resolveBrowserRecordingStopTarget } from "~/browser/browserRecordingScope"; -import { - acquireBrowserSurfaceBackgroundCapture, - useBrowserSurfaceStore, -} from "~/browser/browserSurfaceStore"; +import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; import { isElectron } from "~/env"; import { useEnvironments } from "~/state/environments"; import { previewEnvironment } from "~/state/preview"; @@ -51,7 +48,7 @@ import { previewBridge } from "./previewBridge"; import { isPreviewAutomationTabPresented, revealPreviewAutomationTab, - waitForPreviewAutomationBackgroundPresentation, + withPreviewAutomationBackgroundPresentation, } from "./previewAutomationPresentation"; import { PreviewAutomationNavigationTimeoutError, @@ -63,7 +60,10 @@ import { PreviewAutomationViewportTimeoutError, } from "./previewAutomationErrors"; import { resolvePreviewAutomationOpenWaitPolicy } from "./previewAutomationOpenReadiness"; -import { createPreviewAutomationRequestConsumerAtom } from "./previewAutomationRequestConsumer"; +import { + createPreviewAutomationRequestConsumerAtom, + previewAutomationExecutionBudget, +} from "./previewAutomationRequestConsumer"; import { createPreviewAutomationClientId } from "./previewAutomationClientId"; import { needsPreviewAutomationSessionSync, @@ -136,30 +136,6 @@ const waitForBrowserSurfaceVisibility = async ( }); }; -const withBackgroundAutomationPresentation = async ( - threadRef: ScopedThreadRef, - requestId: string, - tabId: string, - timeoutMs: number, - use: (background: boolean) => Promise, -): Promise => { - const background = !(useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false); - if (!background) return await use(false); - - const releaseCapture = acquireBrowserSurfaceBackgroundCapture(tabId); - try { - await waitForPreviewAutomationBackgroundPresentation({ - threadRef, - requestId, - tabId, - timeoutMs, - }); - return await use(true); - } finally { - releaseCapture(); - } -}; - const waitForNavigationReadiness = async ( threadRef: ScopedThreadRef, requestId: string, @@ -376,7 +352,11 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) const handleRequest = useCallback( async (request: PreviewAutomationRequest): Promise => { const operationDeadline = - Date.now() + Math.max(1, request.timeoutMs - PREVIEW_AUTOMATION_INTERNAL_RESPONSE_GRACE_MS); + Date.now() + + previewAutomationExecutionBudget( + request.timeoutMs, + PREVIEW_AUTOMATION_INTERNAL_RESPONSE_GRACE_MS, + ); const remainingOperationBudget = (requestedTimeoutMs = request.timeoutMs): number => Math.max(1, Math.min(requestedTimeoutMs, operationDeadline - Date.now())); const threadRef: ScopedThreadRef = { @@ -509,7 +489,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) }, ); await ready.bridge.navigate(ready.tabId, resolution.resolvedUrl); - await withBackgroundAutomationPresentation( + await withPreviewAutomationBackgroundPresentation( threadRef, request.requestId, ready.tabId, @@ -560,7 +540,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) case "setColorScheme": { const ready = await requireReadyTab(); const input = request.input as PreviewAutomationSetColorSchemeInput; - await withBackgroundAutomationPresentation( + await withPreviewAutomationBackgroundPresentation( threadRef, request.requestId, ready.tabId, @@ -574,7 +554,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "snapshot": { const ready = await requireReadyTab(); - return await withBackgroundAutomationPresentation( + return await withPreviewAutomationBackgroundPresentation( threadRef, request.requestId, ready.tabId, @@ -584,7 +564,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "click": { const ready = await requireReadyTab(); - return await withBackgroundAutomationPresentation( + return await withPreviewAutomationBackgroundPresentation( threadRef, request.requestId, ready.tabId, @@ -598,7 +578,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "type": { const ready = await requireReadyTab(); - return await withBackgroundAutomationPresentation( + return await withPreviewAutomationBackgroundPresentation( threadRef, request.requestId, ready.tabId, @@ -612,7 +592,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "press": { const ready = await requireReadyTab(); - return await withBackgroundAutomationPresentation( + return await withPreviewAutomationBackgroundPresentation( threadRef, request.requestId, ready.tabId, @@ -626,7 +606,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "scroll": { const ready = await requireReadyTab(); - return await withBackgroundAutomationPresentation( + return await withPreviewAutomationBackgroundPresentation( threadRef, request.requestId, ready.tabId, @@ -640,7 +620,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "evaluate": { const ready = await requireReadyTab(); - return await withBackgroundAutomationPresentation( + return await withPreviewAutomationBackgroundPresentation( threadRef, request.requestId, ready.tabId, @@ -654,7 +634,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "waitFor": { const ready = await requireReadyTab(); - return await withBackgroundAutomationPresentation( + return await withPreviewAutomationBackgroundPresentation( threadRef, request.requestId, ready.tabId, diff --git a/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts b/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts index 0c00954eb37..fa23e18ad56 100644 --- a/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts +++ b/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts @@ -80,4 +80,38 @@ describe("preview automation open readiness", () => { waitForVisibility: false, }); }); + + it("does not require visibility for a reused empty tab", () => { + expect( + resolvePreviewAutomationOpenWaitPolicy( + {} as PreviewAutomationOpenInput, + snapshot({ _tag: "Idle" }), + true, + ), + ).toEqual({ + acknowledgeAfterCreation: false, + waitForOverlay: false, + waitForVisibility: false, + }); + }); + + it("does not require visibility for a reused failed tab", () => { + expect( + resolvePreviewAutomationOpenWaitPolicy( + {} as PreviewAutomationOpenInput, + snapshot({ + _tag: "LoadFailed", + url: "https://example.com/", + title: "Example", + code: -2, + description: "Failed", + }), + true, + ), + ).toEqual({ + acknowledgeAfterCreation: false, + waitForOverlay: true, + waitForVisibility: false, + }); + }); }); diff --git a/apps/web/src/components/preview/previewAutomationOpenReadiness.ts b/apps/web/src/components/preview/previewAutomationOpenReadiness.ts index ca637031d17..3e71daecd78 100644 --- a/apps/web/src/components/preview/previewAutomationOpenReadiness.ts +++ b/apps/web/src/components/preview/previewAutomationOpenReadiness.ts @@ -18,9 +18,13 @@ export function resolvePreviewAutomationOpenWaitPolicy( waitForVisibility: false, }; } + const canPresentBrowserSurface = + input.url !== undefined || + snapshot.navStatus._tag === "Loading" || + snapshot.navStatus._tag === "Success"; return { acknowledgeAfterCreation: false, waitForOverlay: input.url !== undefined || snapshot.navStatus._tag !== "Idle", - waitForVisibility: input.show ?? true, + waitForVisibility: (input.show ?? true) && canPresentBrowserSurface, }; } diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index 8042ef44e31..1bb0b071b11 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -13,6 +13,7 @@ import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelSto import { isPreviewAutomationTabPresented, revealPreviewAutomationTab, + withPreviewAutomationBackgroundPresentation, waitForPreviewAutomationBackgroundPresentation, } from "./previewAutomationPresentation"; @@ -88,4 +89,83 @@ describe("preview automation presentation", () => { vi.useRealTimers(); } }); + + it("accepts a tab that becomes foregrounded while background staging renders", async () => { + vi.useFakeTimers(); + vi.stubGlobal("document", { + querySelectorAll: () => [], + }); + vi.stubGlobal("window", { + setTimeout, + }); + const surface = acquireBrowserSurface("tab-foregrounded"); + try { + revealPreviewAutomationTab(threadRef, "tab-foregrounded"); + const presentation = waitForPreviewAutomationBackgroundPresentation({ + threadRef, + requestId: "request-foregrounded", + tabId: "tab-foregrounded", + timeoutMs: 40, + }); + + surface.present({ x: 0, y: 0, width: 800, height: 600 }, true); + await vi.advanceTimersByTimeAsync(16); + + await expect(presentation).resolves.toBeUndefined(); + } finally { + surface.release(); + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); + + it("releases a background capture lease when the staged operation stalls", async () => { + vi.useFakeTimers(); + vi.stubGlobal("document", { + querySelectorAll: () => [ + { + dataset: { + previewViewport: "tab-background", + previewBackgroundCapture: "true", + }, + offsetWidth: 800, + }, + ], + }); + vi.stubGlobal("window", { + requestAnimationFrame: (callback: FrameRequestCallback) => { + callback(0); + return 1; + }, + }); + try { + const operation = withPreviewAutomationBackgroundPresentation( + threadRef, + "request-stalled", + "tab-background", + 40, + () => new Promise(() => undefined), + ); + const rejection = expect(operation).rejects.toMatchObject({ + _tag: "PreviewAutomationBackgroundPresentationTimeoutError", + requestId: "request-stalled", + tabId: "tab-background", + timeoutMs: 40, + }); + await Promise.resolve(); + expect( + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + ).toBe(1); + + await vi.advanceTimersByTimeAsync(40); + + await rejection; + expect( + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + ).toBeUndefined(); + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); }); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index d111a81e403..9a99b31dafb 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -1,6 +1,9 @@ import type { ScopedThreadRef } from "@t3tools/contracts"; -import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; +import { + acquireBrowserSurfaceBackgroundCapture, + useBrowserSurfaceStore, +} from "~/browser/browserSurfaceStore"; import { setActivePreviewTab } from "~/previewStateStore"; import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelStore"; @@ -28,6 +31,8 @@ export async function waitForPreviewAutomationBackgroundPresentation(input: { }): Promise { const deadline = Date.now() + input.timeoutMs; while (true) { + if (isPreviewAutomationTabPresented(input.threadRef, input.tabId)) return; + const wrapper = Array.from( document.querySelectorAll("[data-preview-viewport]"), ).find( @@ -57,3 +62,50 @@ export async function waitForPreviewAutomationBackgroundPresentation(input: { timeoutMs: input.timeoutMs, }); } + +export async function withPreviewAutomationBackgroundPresentation( + threadRef: ScopedThreadRef, + requestId: string, + tabId: string, + timeoutMs: number, + use: (background: boolean) => Promise, +): Promise { + const background = !(useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false); + if (!background) return await use(false); + + const timeoutError = () => + new PreviewAutomationBackgroundPresentationTimeoutError({ + requestId, + environmentId: threadRef.environmentId, + threadId: threadRef.threadId, + tabId, + timeoutMs, + }); + const releaseCapture = acquireBrowserSurfaceBackgroundCapture(tabId); + let timedOut = false; + let timer: ReturnType | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = globalThis.setTimeout(() => { + timedOut = true; + reject(timeoutError()); + }, timeoutMs); + }); + const operation = (async () => { + await waitForPreviewAutomationBackgroundPresentation({ + threadRef, + requestId, + tabId, + timeoutMs, + }); + if (timedOut) throw timeoutError(); + const stillBackground = !(useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false); + return await use(stillBackground); + })(); + + try { + return await Promise.race([operation, deadline]); + } finally { + if (timer !== undefined) globalThis.clearTimeout(timer); + releaseCapture(); + } +} diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts index 089a39cf450..9a9a101705b 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts @@ -19,6 +19,7 @@ import { } from "./previewAutomationErrors"; import { createPreviewAutomationRequestConsumerAtom, + previewAutomationExecutionBudget, serializePreviewAutomationError, } from "./previewAutomationRequestConsumer"; @@ -56,6 +57,12 @@ const consumerState = (handleRequest: (request: PreviewAutomationRequest) => Pro }); describe("previewAutomationRequestConsumer", () => { + it("preserves the full execution budget for short requested timeouts", () => { + expect(previewAutomationExecutionBudget(100, 250)).toBe(100); + expect(previewAutomationExecutionBudget(500, 250)).toBe(500); + expect(previewAutomationExecutionBudget(1_000, 250)).toBe(750); + }); + it("acknowledges a replacement stream before consuming requests from it", async () => { const requestsAtom = Atom.make( AsyncResult.success({ @@ -423,6 +430,56 @@ describe("previewAutomationRequestConsumer", () => { } }); + it("does not collapse a short host timeout to one millisecond", async () => { + vi.useFakeTimers(); + try { + const requestsAtom = Atom.make>( + AsyncResult.initial(false), + ); + const respond = vi.fn(async (_response: PreviewAutomationResponse) => undefined); + const state = consumerState(() => new Promise(() => undefined)); + const consumerAtom = createPreviewAutomationRequestConsumerAtom({ + requestsAtom, + clientId, + connectionAtom: state.connectionAtom, + environmentId, + requestHandlerAtom: state.requestHandlerAtom, + respond, + label: "test:preview-automation-short-deadline", + }); + const registry = AtomRegistry.make(); + registry.mount(consumerAtom); + registry.set( + requestsAtom, + AsyncResult.success( + requestEvent("request-short", { + operation: "click", + tabId, + timeoutMs: 100, + }), + ), + ); + + await vi.advanceTimersByTimeAsync(99); + expect(respond).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + + expect(respond).toHaveBeenCalledWith( + expect.objectContaining({ + requestId: "request-short", + ok: false, + error: expect.objectContaining({ + _tag: "PreviewAutomationTimeoutError", + detail: expect.objectContaining({ timeoutMs: 100 }), + }), + }), + ); + registry.dispose(); + } finally { + vi.useRealTimers(); + } + }); + it("maps desktop non-editable targets to the public typed response", () => { expect( serializePreviewAutomationError( diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts index 3c508886d5b..92688862b90 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts @@ -17,6 +17,11 @@ type AutomationStreamResult = AsyncResult.AsyncResult (timeoutMs > responseGraceMs * 2 ? timeoutMs - responseGraceMs : timeoutMs); + const handleWithinResponseBudget = ( request: PreviewAutomationRequest, environmentId: PreviewAutomationHost["environmentId"], @@ -24,7 +29,10 @@ const handleWithinResponseBudget = ( ): Promise => new Promise((resolve, reject) => { let settled = false; - const timeoutMs = Math.max(1, request.timeoutMs - PREVIEW_AUTOMATION_RESPONSE_GRACE_MS); + const timeoutMs = previewAutomationExecutionBudget( + request.timeoutMs, + PREVIEW_AUTOMATION_RESPONSE_GRACE_MS, + ); const timer = globalThis.setTimeout(() => { if (settled) return; settled = true; From e9472ba3598ef77c63b740b131e2a8d2139c704b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 25 Jul 2026 22:56:18 +0100 Subject: [PATCH 07/43] Harden preview capture and deadline fallbacks Preserve control deadlines and screenshot recovery. Default legacy snapshot IPC and scope loopback host overrides. --- apps/desktop/src/ipc/methods/preview.ts | 2 +- apps/desktop/src/preload.ts | 2 +- apps/desktop/src/preview/Manager.test.ts | 51 ++++++-- apps/desktop/src/preview/Manager.ts | 115 ++++++++++++------ apps/web/src/browser/HostedBrowserWebview.tsx | 15 +-- .../browser/hostedBrowserWebviewStyle.test.ts | 2 +- .../preview/PreviewAutomationHosts.tsx | 3 +- .../previewAutomationPresentation.test.ts | 21 ++-- .../preview/previewAutomationPresentation.ts | 27 ++-- packages/contracts/src/ipc.test.ts | 17 ++- packages/contracts/src/ipc.ts | 3 +- scripts/dev-runner.ts | 2 +- 12 files changed, 171 insertions(+), 89 deletions(-) diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 97ddd8be0d4..3483d5a5339 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -4,8 +4,8 @@ import { DesktopPreviewAutomationClickInputSchema, DesktopPreviewAutomationEvaluateInputSchema, DesktopPreviewAutomationPressInputSchema, - DesktopPreviewAutomationSnapshotInputSchema, DesktopPreviewAutomationScrollInputSchema, + DesktopPreviewAutomationSnapshotInputSchema, DesktopPreviewAutomationTypeInputSchema, DesktopPreviewAutomationWaitForInputSchema, DesktopPreviewConfigInputSchema, diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 035aaef2361..c7b76ead74e 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -203,7 +203,7 @@ contextBridge.exposeInMainWorld("desktopBridge", { automation: { status: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_STATUS_CHANNEL, { tabId }), - snapshot: (tabId, background) => + snapshot: (tabId, background = false) => ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, { tabId, background, diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index f087c0b1443..a4c8d422d66 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -164,7 +164,7 @@ describe("PreviewManager", () => { }; createFromBuffer.mockReturnValue(image); let attached = false; - let captureAvailable = true; + let captureMode: "available" | "failure" | "timeout" = "available"; const attach = vi.fn(() => { attached = true; }); @@ -192,7 +192,8 @@ describe("PreviewManager", () => { return { nodes: [] }; } if (method === "Page.captureScreenshot") { - if (!captureAvailable) throw new Error("UnknownVizError"); + if (captureMode === "failure") throw new Error("UnknownVizError"); + if (captureMode === "timeout") return await new Promise(() => undefined); return { data: png.toString("base64") }; } return undefined; @@ -254,6 +255,8 @@ describe("PreviewManager", () => { fromSurface: true, captureBeyondViewport: false, }); + expect(sendCommand).not.toHaveBeenCalledWith("Page.bringToFront", undefined); + expect(focus).not.toHaveBeenCalled(); const backgroundCdpCaptured = yield* manager.automationSnapshot("tab_snapshot", true); expect(backgroundCdpCaptured.screenshot).toMatchObject({ width: 640, height: 360 }); @@ -267,7 +270,7 @@ describe("PreviewManager", () => { expect(restoreFocus).toHaveBeenCalledOnce(); expect(capturePage).not.toHaveBeenCalled(); - captureAvailable = false; + captureMode = "failure"; const backgroundFallbackCaptured = yield* manager.automationSnapshot( "tab_snapshot", true, @@ -279,15 +282,37 @@ describe("PreviewManager", () => { expect(capturePage).toHaveBeenCalledWith(undefined, { stayHidden: false }); capturePage.mockClear(); + const foregroundFallbackCaptured = yield* manager.automationSnapshot("tab_snapshot"); + expect(foregroundFallbackCaptured.screenshot).toMatchObject({ + width: 640, + height: 360, + }); + expect(capturePage).toHaveBeenCalledOnce(); + + capturePage.mockClear(); + capturePage.mockResolvedValueOnce({ ...image, isEmpty: () => true }); const degraded = yield* manager.automationSnapshot("tab_snapshot"); expect(degraded.screenshot).toBeNull(); - expect(capturePage).not.toHaveBeenCalled(); - expect(detach).toHaveBeenCalledTimes(2); + expect(capturePage).toHaveBeenCalledOnce(); + expect(detach).not.toHaveBeenCalled(); + + capturePage.mockClear(); + captureMode = "timeout"; + const timedOutCapture = yield* manager + .automationSnapshot("tab_snapshot") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(5_000); + expect((yield* Fiber.join(timedOutCapture)).screenshot).toMatchObject({ + width: 640, + height: 360, + }); + expect(capturePage).toHaveBeenCalledOnce(); + expect(detach).toHaveBeenCalledOnce(); - captureAvailable = true; + captureMode = "available"; const recovered = yield* manager.automationSnapshot("tab_snapshot"); expect(recovered.screenshot).toMatchObject({ width: 640, height: 360 }); - expect(attach).toHaveBeenCalledTimes(3); + expect(attach).toHaveBeenCalledTimes(2); }), ), ); @@ -412,6 +437,12 @@ describe("PreviewManager", () => { yield* manager.createTab("tab_queued_timeout"); yield* manager.registerWebview("tab_queued_timeout", 44); + let latestController: string | undefined; + yield* manager.subscribeStateChanges((tabId, state) => + Effect.sync(() => { + if (tabId === "tab_queued_timeout") latestController = state.controller; + }), + ); const active = yield* manager .automationEvaluate("tab_queued_timeout", { expression: "document.title" }) .pipe(Effect.forkChild({ startImmediately: true })); @@ -421,13 +452,15 @@ describe("PreviewManager", () => { .pipe(Effect.forkChild({ startImmediately: true })); yield* Effect.yieldNow; - yield* TestClock.adjust(1_000); + yield* TestClock.adjust(1_250); expect(Exit.isFailure(yield* Effect.exit(Fiber.join(queued)))).toBe(true); expect(detach).not.toHaveBeenCalled(); + expect(latestController).toBe("agent"); - yield* TestClock.adjust(14_000); + yield* TestClock.adjust(13_500); expect(Exit.isFailure(yield* Effect.exit(Fiber.join(active)))).toBe(true); expect(detach).toHaveBeenCalledOnce(); + expect(latestController).toBe("none"); expect( yield* manager.automationEvaluate("tab_queued_timeout", { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 53c82c9dd56..d2f1466decb 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -106,6 +106,10 @@ const AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS = 250; const AUTOMATION_SCREENSHOT_TIMEOUT_MS = 5_000; const AUTOMATION_BACKGROUND_CDP_SCREENSHOT_TIMEOUT_MS = 2_000; const AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS = 3_000; +const automationExecutionBudget = (timeoutMs: number): number => + timeoutMs > AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS * 2 + ? timeoutMs - AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS + : timeoutMs; const DIAGNOSTIC_BUFFER_LIMIT = 200; const MAX_ARTIFACT_SITE_SLUG_LENGTH = 80; const AGENT_CURSOR_MOVE_MS = 160; @@ -945,6 +949,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } return yield* use(send, sendCleanup); }); + let detachOnTimeout = true; + let permitAcquired = false; const finalize = Effect.fn("PreviewManager.finalizeControlAction")(function* ( exit: Exit.Exit, ) { @@ -974,10 +980,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function error: errorMessage, }); } - const tabs = yield* SynchronizedRef.get(tabsRef); - if (tabs.has(tabId)) yield* update(tabId, { controller: "none" }); + if (permitAcquired) { + const tabs = yield* SynchronizedRef.get(tabsRef); + if (tabs.has(tabId)) yield* update(tabId, { controller: "none" }); + } }); - let detachOnTimeout = true; const boundedExecution = Effect.gen(function* () { // Session initialization itself sends CDP commands. Keep it inside the // operation deadline so an offscreen or suspended guest cannot retain @@ -987,10 +994,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return yield* control.semaphore.withPermit( Effect.sync(() => { detachOnTimeout = true; + permitAcquired = true; }).pipe(Effect.andThen(execute())), ); }).pipe( - Effect.timeoutOption(Math.max(1, timeoutMs - AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS)), + Effect.timeoutOption(automationExecutionBudget(timeoutMs)), Effect.flatMap((result) => Option.isNone(result) ? Effect.fail(new PreviewAutomationTimeoutError({ tabId, timeoutMs })) @@ -1269,6 +1277,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const beforeInput = (event: Electron.Event, input: Electron.Input): void => { runFork(forwardShortcut(event, input)); }; + const devtoolsClosed = (): void => { + runFork( + Effect.gen(function* () { + const current = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if ( + current?.webContentsId !== wc.id || + current.colorScheme === "system" || + wc.isDestroyed() + ) { + return; + } + yield* restoreControlSession(tabId, wc); + }), + ); + }; yield* Scope.addFinalizer( scope, attempt({ operation: "detachListeners", tabId, webContentsId: wc.id }, () => { @@ -1279,6 +1302,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc.off("did-stop-loading", sync); wc.off("did-fail-load", failed as never); wc.off("before-input-event", beforeInput); + wc.off("devtools-closed", devtoolsClosed); wc.ipc.off(HUMAN_INPUT_CHANNEL, humanInput); }).pipe(Effect.ignore), ); @@ -1290,6 +1314,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc.on("did-start-loading", sync); wc.on("did-stop-loading", sync); wc.on("did-fail-load", failed as never); + wc.on("devtools-closed", devtoolsClosed); wc.ipc.on(HUMAN_INPUT_CHANNEL, humanInput); wc.setWindowOpenHandler(({ url }) => { runFork( @@ -1765,7 +1790,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function Math.max(1, DEFAULT_AUTOMATION_TIMEOUT_MS - AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS), ), Effect.flatMap((result) => - Option.isSome(result) ? Effect.void : detachControlSession(wc.id), + Option.isSome(result) + ? Effect.void + : Effect.logWarning("Timed out restoring the preview control session.", { + tabId, + webContentsId: wc.id, + }).pipe(Effect.andThen(detachControlSession(wc.id))), ), Effect.ignore, ); @@ -1976,6 +2006,14 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function sourceSize.width > MAX_SCREENSHOT_WIDTH ? sourceImage.resize({ width: MAX_SCREENSHOT_WIDTH }) : sourceImage; + if (image.isEmpty()) { + return yield* new PreviewOperationError({ + operation, + tabId, + webContentsId: wc.id, + cause: new Error("Screenshot resize returned an invalid PNG"), + }); + } const size = image.getSize(); return { mimeType: "image/png" as const, @@ -2130,6 +2168,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function Exit.isSuccess(cdpScreenshotResult) && Option.isSome(cdpScreenshotResult.value) ? cdpScreenshotResult.value.value : null; + const detachAfterCapture = + Exit.isSuccess(cdpScreenshotResult) && Option.isNone(cdpScreenshotResult.value); if (screenshot === null) { const cdpFailure = Exit.isFailure(cdpScreenshotResult) ? cdpScreenshotResult.cause @@ -2137,32 +2177,22 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId, timeoutMs: cdpScreenshotTimeoutMs, }); - // A timed-out debugger command may still settle after its Effect has - // been interrupted. Detach the session so later automation starts - // from a fresh CDP connection instead of inheriting that command. - yield* detachControlSession(wc.id); - const backgroundScreenshotResult = background - ? yield* captureBackgroundPage(tabId, wc).pipe( - Effect.timeoutOption(AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS), - Effect.exit, - ) - : null; + const backgroundScreenshotResult = yield* captureBackgroundPage(tabId, wc).pipe( + Effect.timeoutOption(AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS), + Effect.exit, + ); screenshot = - backgroundScreenshotResult !== null && Exit.isSuccess(backgroundScreenshotResult) && Option.isSome(backgroundScreenshotResult.value) ? backgroundScreenshotResult.value.value : null; if (screenshot === null) { - const backgroundFailure = - backgroundScreenshotResult === null - ? null - : Exit.isFailure(backgroundScreenshotResult) - ? backgroundScreenshotResult.cause - : new PreviewAutomationTimeoutError({ - tabId, - timeoutMs: AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS, - }); + const backgroundFailure = Exit.isFailure(backgroundScreenshotResult) + ? backgroundScreenshotResult.cause + : new PreviewAutomationTimeoutError({ + tabId, + timeoutMs: AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS, + }); yield* Effect.logWarning("Preview automation screenshot capture was unavailable.", { tabId, webContentsId: wc.id, @@ -2180,6 +2210,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function networkEntries: [...(browserDiagnostics?.networkEntries ?? [])], actionTimeline: [...(timelines.get(tabId) ?? [])], screenshot, + detachAfterCapture, }; }, ); @@ -2190,15 +2221,18 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ) { const wc = yield* requireWebContents(tabId); if (!background) { - return yield* withControlSession(tabId, wc, "snapshot", (send) => + const result = yield* withControlSession(tabId, wc, "snapshot", (send) => captureAutomationSnapshot(tabId, wc, send, false), ); + if (result.detachAfterCapture) yield* detachControlSession(wc.id); + const { detachAfterCapture: _, ...snapshot } = result; + return snapshot; } const previouslyFocused = yield* attempt( { operation: "automationSnapshot.getFocusedWebContents", tabId, webContentsId: wc.id }, () => webContents.getFocusedWebContents(), - ); + ).pipe(Effect.orElseSucceed(() => null)); const restoreFocus = previouslyFocused && previouslyFocused.id !== wc.id ? attempt( @@ -2218,18 +2252,19 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function // The renderer stages it transparently while this short capture lease // activates the guest itself. Restoring the prior WebContents preserves // both application focus and the user-visible preview selection. - return yield* Effect.gen(function* () { - yield* attempt( - { operation: "automationSnapshot.focusWebContents", tabId, webContentsId: wc.id }, - () => wc.focus(), - ); - return yield* withControlSession(tabId, wc, "snapshot", (send) => - Effect.gen(function* () { - yield* send("Page.bringToFront"); - return yield* captureAutomationSnapshot(tabId, wc, send, true); - }), - ); - }).pipe(Effect.ensuring(restoreFocus)); + const result = yield* withControlSession(tabId, wc, "snapshot", (send) => + Effect.gen(function* () { + yield* attempt( + { operation: "automationSnapshot.focusWebContents", tabId, webContentsId: wc.id }, + () => wc.focus(), + ); + yield* send("Page.bringToFront"); + return yield* captureAutomationSnapshot(tabId, wc, send, true); + }), + ).pipe(Effect.ensuring(restoreFocus)); + if (result.detachAfterCapture) yield* detachControlSession(wc.id); + const { detachAfterCapture: _, ...snapshot } = result; + return snapshot; }); const resolveClickPoint = Effect.fn("PreviewManager.resolveClickPoint")(function* ( @@ -2725,7 +2760,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc, "waitFor", (send) => performAutomationWaitFor(tabId, input, send), - input.timeoutMs, + (input.timeoutMs ?? DEFAULT_AUTOMATION_TIMEOUT_MS) + AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS * 2, ); }); diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index 1a499f13f13..8bff460417d 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -56,17 +56,14 @@ export function HostedBrowserWebview(props: { useShallow((state) => { const current = state.byTabId[tabId]; const backgroundCapture = (state.backgroundCaptureCountByTabId[tabId] ?? 0) > 0; - const panelRect = resolveBrowserSurfacePanelRect(state.byTabId, tabId); return { backgroundCapture, - rect: - panelRect ?? - (backgroundCapture - ? resolveBrowserSurfaceBackgroundCaptureRect(state.byTabId, tabId, { - width: window.innerWidth, - height: window.innerHeight, - }) - : null), + rect: backgroundCapture + ? resolveBrowserSurfaceBackgroundCaptureRect(state.byTabId, tabId, { + width: window.innerWidth, + height: window.innerHeight, + }) + : resolveBrowserSurfacePanelRect(state.byTabId, tabId), visible: current?.visible ?? false, }; }), diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts index 88aa3629b26..290aa58c0fc 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts @@ -26,7 +26,7 @@ describe("resolveHostedBrowserWebviewWrapperStyle", () => { }); }); - it("places a background capture target beneath the active browser surface", () => { + it("places a nearly transparent background target above the active surface", () => { expect( resolveHostedBrowserWebviewWrapperStyle({ active: false, diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 0b23ea14847..5dfb7001a4d 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -104,6 +104,7 @@ const waitForBrowserSurfaceVisibility = async ( timeoutMs: number, ): Promise => { const deadline = Date.now() + timeoutMs; + const requiredStableMs = Math.min(100, Math.max(0, timeoutMs - 50)); let presentedSince: number | null = null; while (Date.now() <= deadline) { const now = Date.now(); @@ -112,7 +113,7 @@ const waitForBrowserSurfaceVisibility = async ( // Require the selection to survive multiple presentation updates. A // single transient `visible` frame can otherwise make open acknowledge // just before routing or panel reconciliation unmounts the surface. - if (now - presentedSince >= 100) return; + if (now - presentedSince >= requiredStableMs) return; } else { presentedSince = null; // Session reconciliation and route hydration can race a cold open. diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index 1bb0b071b11..fbbb7dbb63a 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -119,7 +119,7 @@ describe("preview automation presentation", () => { } }); - it("releases a background capture lease when the staged operation stalls", async () => { + it("keeps the background capture lease until a staged operation settles", async () => { vi.useFakeTimers(); vi.stubGlobal("document", { querySelectorAll: () => [ @@ -139,27 +139,28 @@ describe("preview automation presentation", () => { }, }); try { + let resolveOperation!: (value: string) => void; const operation = withPreviewAutomationBackgroundPresentation( threadRef, "request-stalled", "tab-background", 40, - () => new Promise(() => undefined), + () => + new Promise((resolve) => { + resolveOperation = resolve; + }), ); - const rejection = expect(operation).rejects.toMatchObject({ - _tag: "PreviewAutomationBackgroundPresentationTimeoutError", - requestId: "request-stalled", - tabId: "tab-background", - timeoutMs: 40, - }); await Promise.resolve(); expect( useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], ).toBe(1); await vi.advanceTimersByTimeAsync(40); - - await rejection; + expect( + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + ).toBe(1); + resolveOperation("captured"); + await expect(operation).resolves.toBe("captured"); expect( useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], ).toBeUndefined(); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index 9a99b31dafb..ee37f1339ca 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -82,28 +82,27 @@ export async function withPreviewAutomationBackgroundPresentation( timeoutMs, }); const releaseCapture = acquireBrowserSurfaceBackgroundCapture(tabId); - let timedOut = false; let timer: ReturnType | undefined; const deadline = new Promise((_resolve, reject) => { timer = globalThis.setTimeout(() => { - timedOut = true; reject(timeoutError()); }, timeoutMs); }); - const operation = (async () => { - await waitForPreviewAutomationBackgroundPresentation({ - threadRef, - requestId, - tabId, - timeoutMs, - }); - if (timedOut) throw timeoutError(); - const stillBackground = !(useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false); - return await use(stillBackground); - })(); + const staging = waitForPreviewAutomationBackgroundPresentation({ + threadRef, + requestId, + tabId, + timeoutMs, + }); try { - return await Promise.race([operation, deadline]); + await Promise.race([staging, deadline]); + if (timer !== undefined) { + globalThis.clearTimeout(timer); + timer = undefined; + } + const stillBackground = !(useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false); + return await use(stillBackground); } finally { if (timer !== undefined) globalThis.clearTimeout(timer); releaseCapture(); diff --git a/packages/contracts/src/ipc.test.ts b/packages/contracts/src/ipc.test.ts index 20db75368a9..0564adf0fc7 100644 --- a/packages/contracts/src/ipc.test.ts +++ b/packages/contracts/src/ipc.test.ts @@ -1,7 +1,10 @@ import * as Schema from "effect/Schema"; import { describe, expect, it } from "vite-plus/test"; -import { DesktopEnvironmentBootstrapSchema } from "./ipc.ts"; +import { + DesktopEnvironmentBootstrapSchema, + DesktopPreviewAutomationSnapshotInputSchema, +} from "./ipc.ts"; describe("DesktopEnvironmentBootstrapSchema", () => { const decode = Schema.decodeUnknownSync(DesktopEnvironmentBootstrapSchema); @@ -36,3 +39,15 @@ describe("DesktopEnvironmentBootstrapSchema", () => { ).toBeNull(); }); }); + +describe("DesktopPreviewAutomationSnapshotInputSchema", () => { + const decode = Schema.decodeUnknownSync(DesktopPreviewAutomationSnapshotInputSchema); + + it("defaults omitted and undefined background flags to foreground capture", () => { + expect(decode({ tabId: "tab-1" })).toEqual({ tabId: "tab-1", background: false }); + expect(decode({ tabId: "tab-1", background: undefined })).toEqual({ + tabId: "tab-1", + background: false, + }); + }); +}); diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index c357c3b5863..f0792e5344d 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -58,6 +58,7 @@ import type { TerminalWriteInput, } from "./terminal.ts"; import type { ServerRemoveKeybindingInput, ServerUpsertKeybindingInput } from "./server.ts"; +import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import type { DiscoveredLocalServerList, @@ -911,7 +912,7 @@ export const DesktopPreviewTabInputSchema = Schema.Struct({ export const DesktopPreviewAutomationSnapshotInputSchema = Schema.Struct({ tabId: DesktopPreviewTabIdSchema, - background: Schema.Boolean, + background: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), }); export const DesktopPreviewRegisterWebviewInputSchema = Schema.Struct({ diff --git a/scripts/dev-runner.ts b/scripts/dev-runner.ts index b3506140903..ea365c617d0 100644 --- a/scripts/dev-runner.ts +++ b/scripts/dev-runner.ts @@ -304,7 +304,7 @@ export function createDevRunnerEnv({ delete output.T3CODE_DESKTOP_WS_URL; } - if (mode !== "dev:server") { + if (mode === "dev" || mode === "dev:web" || mode === "dev:desktop") { output.HOST = DEV_LOOPBACK_HOST; } From e6cfcc60a9cbfacede104e0fbcb609f98a1b83b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 25 Jul 2026 23:11:44 +0100 Subject: [PATCH 08/43] Keep preview automation within request deadlines - Release background capture leases when staged operations time out - Keep desktop wait operations inside caller timeout budgets --- apps/desktop/src/preview/Manager.test.ts | 4 +-- apps/desktop/src/preview/Manager.ts | 2 +- .../previewAutomationPresentation.test.ts | 20 +++++++------- .../preview/previewAutomationPresentation.ts | 27 ++++++++++--------- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index a4c8d422d66..4e8b843653e 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -452,12 +452,12 @@ describe("PreviewManager", () => { .pipe(Effect.forkChild({ startImmediately: true })); yield* Effect.yieldNow; - yield* TestClock.adjust(1_250); + yield* TestClock.adjust(750); expect(Exit.isFailure(yield* Effect.exit(Fiber.join(queued)))).toBe(true); expect(detach).not.toHaveBeenCalled(); expect(latestController).toBe("agent"); - yield* TestClock.adjust(13_500); + yield* TestClock.adjust(14_000); expect(Exit.isFailure(yield* Effect.exit(Fiber.join(active)))).toBe(true); expect(detach).toHaveBeenCalledOnce(); expect(latestController).toBe("none"); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index d2f1466decb..da4a851dde5 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2760,7 +2760,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc, "waitFor", (send) => performAutomationWaitFor(tabId, input, send), - (input.timeoutMs ?? DEFAULT_AUTOMATION_TIMEOUT_MS) + AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS * 2, + input.timeoutMs ?? DEFAULT_AUTOMATION_TIMEOUT_MS, ); }); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index fbbb7dbb63a..ae0a673ff15 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -119,7 +119,7 @@ describe("preview automation presentation", () => { } }); - it("keeps the background capture lease until a staged operation settles", async () => { + it("releases a background capture lease when the staged operation stalls", async () => { vi.useFakeTimers(); vi.stubGlobal("document", { querySelectorAll: () => [ @@ -139,28 +139,26 @@ describe("preview automation presentation", () => { }, }); try { - let resolveOperation!: (value: string) => void; const operation = withPreviewAutomationBackgroundPresentation( threadRef, "request-stalled", "tab-background", 40, - () => - new Promise((resolve) => { - resolveOperation = resolve; - }), + () => new Promise(() => undefined), ); + const rejection = expect(operation).rejects.toMatchObject({ + _tag: "PreviewAutomationBackgroundPresentationTimeoutError", + requestId: "request-stalled", + tabId: "tab-background", + timeoutMs: 40, + }); await Promise.resolve(); expect( useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], ).toBe(1); await vi.advanceTimersByTimeAsync(40); - expect( - useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], - ).toBe(1); - resolveOperation("captured"); - await expect(operation).resolves.toBe("captured"); + await rejection; expect( useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], ).toBeUndefined(); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index ee37f1339ca..9a99b31dafb 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -82,27 +82,28 @@ export async function withPreviewAutomationBackgroundPresentation( timeoutMs, }); const releaseCapture = acquireBrowserSurfaceBackgroundCapture(tabId); + let timedOut = false; let timer: ReturnType | undefined; const deadline = new Promise((_resolve, reject) => { timer = globalThis.setTimeout(() => { + timedOut = true; reject(timeoutError()); }, timeoutMs); }); - const staging = waitForPreviewAutomationBackgroundPresentation({ - threadRef, - requestId, - tabId, - timeoutMs, - }); - - try { - await Promise.race([staging, deadline]); - if (timer !== undefined) { - globalThis.clearTimeout(timer); - timer = undefined; - } + const operation = (async () => { + await waitForPreviewAutomationBackgroundPresentation({ + threadRef, + requestId, + tabId, + timeoutMs, + }); + if (timedOut) throw timeoutError(); const stillBackground = !(useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false); return await use(stillBackground); + })(); + + try { + return await Promise.race([operation, deadline]); } finally { if (timer !== undefined) globalThis.clearTimeout(timer); releaseCapture(); From a6134bf5c0d048a04fa3d659428643339ce2f917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sat, 25 Jul 2026 23:36:38 +0100 Subject: [PATCH 09/43] Document preview automation branch behavior --- BRANCH_DETAILS.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 BRANCH_DETAILS.md diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md new file mode 100644 index 00000000000..7eda4099a13 --- /dev/null +++ b/BRANCH_DETAILS.md @@ -0,0 +1,51 @@ +# Preview Automation Reliability + +Product-native preview automation remains bounded and recoverable across the web host, MCP server, and Electron CDP controller. Preserve this behavior until upstream provides equivalent operation deadlines, control-session recovery, and degraded semantic snapshots. + +Expected behavior: + +- Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session when initialization or an acquired permit may have left a CDP command pending. A request that times out while queued behind another action does not detach that action's shared debugger session. +- Click, type, and wait operations propagate their caller-supplied timeout into the desktop control-session boundary. Operations without a caller timeout retain the bounded desktop default. +- Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer identifies the request as background automation and stages the still-mounted guest above the selected browser surface at effectively transparent opacity for two compositor frames. The same short presentation lease covers navigation readiness, color-scheme changes, evaluation, waiting, and pointer or keyboard interactions so every CDP-backed operation can target an unselected tab without changing the right-panel selection. The desktop manager then temporarily focuses a background snapshot guest, brings its CDP page to the foreground, captures its compositor surface, and best-effort restores the previously focused WebContents when it is still available. The staged guest cannot receive pointer input. A separately bounded `webContents.capturePage` attempt provides a fallback for foreground and background snapshots when CDP is unavailable. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. +- Desktop preview guests no longer create their CDP debugger session eagerly when a webview registers. Session initialization is lazy and included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. Closing detached DevTools restores an explicit color-scheme override through the separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. +- Inactive preview webviews remain mounted, retain their declared viewport, and stay CSS-visible while positioned outside the human-visible panel. This preserves their runtime and input automation without selecting them. Background capture uses a reference-counted presentation lease and always restores the offscreen position afterward; it does not change the right-panel tab selected by the user. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target while staging is pending, that visible presentation satisfies readiness. A never-presented tab no longer depends on another browser surface having supplied a panel rectangle: capture staging falls back to a deterministic rectangle fitted inside the renderer viewport. +- The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. +- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to show the browser use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts report whether the right panel was open, which surface was active, whether the requested browser surface was registered, and whether it had a presentation rectangle. +- A newly created preview tab applies its server snapshot and assigned tab id, initiates any requested selection, and acknowledges server-side creation immediately without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching right-panel surface, then waits for stable panel presentation; while the request remains pending it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. +- The standard dev runner uses `127.0.0.1` consistently for its generated web, HTTP, and WebSocket loopback URLs and pins Vite to the same host. It sets the generic `HOST` override only for modes that launch the composite stack, web, or desktop, leaving server-only invocations untouched. Environment-port navigation therefore cannot resolve the backend as IPv4 while the related Vite listener is reachable only through IPv6 `localhost`. +- The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. + +Current limitations: + +- Electron screenshot capture can still be unavailable when both the CDP view capture and bounded Electron `capturePage` fallback fail. The intended degraded result remains a usable semantic snapshot with `screenshot: null`, not raster evidence. + +Primary files: + +- `apps/desktop/src/preview/Manager.ts` +- `apps/desktop/src/ipc/methods/preview.ts` +- `apps/desktop/src/preload.ts` +- `apps/server/src/mcp/McpHttpServer.ts` +- `apps/server/src/mcp/toolkits/preview/tools.ts` +- `apps/web/src/browser/HostedBrowserWebview.tsx` +- `apps/web/src/browser/browserSurfaceStore.ts` +- `apps/web/src/browser/hostedBrowserWebviewStyle.ts` +- `apps/web/src/components/auth/PairingRouteSurface.tsx` +- `apps/web/src/components/preview/PreviewAutomationHosts.tsx` +- `apps/web/src/components/preview/previewAutomationPresentation.ts` +- `apps/web/src/components/preview/previewAutomationOpenReadiness.ts` +- `apps/web/src/components/preview/previewAutomationErrors.ts` +- `apps/web/src/components/preview/previewAutomationRequestConsumer.ts` +- `packages/contracts/src/previewAutomation.ts` +- `packages/contracts/src/ipc.ts` +- `scripts/dev-runner.ts` + +Focused regression coverage lives in `scripts/dev-runner.test.ts`, `apps/desktop/src/preview/Manager.test.ts`, `apps/server/src/mcp/McpHttpServer.test.ts`, `apps/web/src/browser/browserSurfaceStore.test.ts`, `apps/web/src/browser/hostedBrowserWebviewStyle.test.ts`, `apps/web/src/components/auth/PairingRouteSurface.logic.test.ts`, `apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationPresentation.test.ts`, `apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts`, `packages/contracts/src/ipc.test.ts`, and `packages/contracts/src/preview.test.ts`. + +```sh +vp test run scripts/dev-runner.test.ts apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/web/src/browser/browserSurfaceStore.test.ts apps/web/src/browser/hostedBrowserWebviewStyle.test.ts apps/web/src/components/auth/PairingRouteSurface.logic.test.ts apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts apps/web/src/components/preview/previewAutomationPresentation.test.ts apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts packages/contracts/src/ipc.test.ts packages/contracts/src/preview.test.ts +``` + +## Development Ports + +- Web: `5744` +- Server/WebSocket: `13784` From 01debc6827bf149923578c9403c16f17e37beba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sun, 26 Jul 2026 14:19:18 +0100 Subject: [PATCH 10/43] Avoid focusing background guests during preview snapshots --- BRANCH_DETAILS.md | 6 +- apps/desktop/src/preview/Manager.test.ts | 19 +--- apps/desktop/src/preview/Manager.ts | 43 ++------- .../preview/PreviewAutomationHosts.tsx | 87 ++++--------------- 4 files changed, 31 insertions(+), 124 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 7eda4099a13..fbe31efa63d 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -6,9 +6,9 @@ Expected behavior: - Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session when initialization or an acquired permit may have left a CDP command pending. A request that times out while queued behind another action does not detach that action's shared debugger session. - Click, type, and wait operations propagate their caller-supplied timeout into the desktop control-session boundary. Operations without a caller timeout retain the bounded desktop default. -- Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer identifies the request as background automation and stages the still-mounted guest above the selected browser surface at effectively transparent opacity for two compositor frames. The same short presentation lease covers navigation readiness, color-scheme changes, evaluation, waiting, and pointer or keyboard interactions so every CDP-backed operation can target an unselected tab without changing the right-panel selection. The desktop manager then temporarily focuses a background snapshot guest, brings its CDP page to the foreground, captures its compositor surface, and best-effort restores the previously focused WebContents when it is still available. The staged guest cannot receive pointer input. A separately bounded `webContents.capturePage` attempt provides a fallback for foreground and background snapshots when CDP is unavailable. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. +- Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests no longer create their CDP debugger session eagerly when a webview registers. Session initialization is lazy and included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. Closing detached DevTools restores an explicit color-scheme override through the separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. -- Inactive preview webviews remain mounted, retain their declared viewport, and stay CSS-visible while positioned outside the human-visible panel. This preserves their runtime and input automation without selecting them. Background capture uses a reference-counted presentation lease and always restores the offscreen position afterward; it does not change the right-panel tab selected by the user. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target while staging is pending, that visible presentation satisfies readiness. A never-presented tab no longer depends on another browser surface having supplied a panel rectangle: capture staging falls back to a deterministic rectangle fitted inside the renderer viewport. +- Inactive preview webviews remain mounted, retain their declared viewport, and stay CSS-visible while positioned outside the human-visible panel. This preserves their runtime and semantic or input automation without selecting them. Background snapshot capture uses a reference-counted presentation lease and always restores the offscreen position afterward; navigation, color-scheme changes, evaluation, waits, and input operations remain offscreen and do not acquire that native-surface lease. Snapshot staging does not change the right-panel tab selected by the user. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: capture staging falls back to a deterministic rectangle fitted inside the renderer viewport. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to show the browser use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts report whether the right panel was open, which surface was active, whether the requested browser surface was registered, and whether it had a presentation rectangle. - A newly created preview tab applies its server snapshot and assigned tab id, initiates any requested selection, and acknowledges server-side creation immediately without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching right-panel surface, then waits for stable panel presentation; while the request remains pending it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. @@ -17,7 +17,7 @@ Expected behavior: Current limitations: -- Electron screenshot capture can still be unavailable when both the CDP view capture and bounded Electron `capturePage` fallback fail. The intended degraded result remains a usable semantic snapshot with `screenshot: null`, not raster evidence. +- Electron screenshot capture can still be unavailable when both the bounded CDP compositor capture and hidden `capturePage` fallback fail. The intended degraded result remains a usable semantic snapshot with `screenshot: null`, not raster evidence. Primary files: diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 4e8b843653e..e76c0089202 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -171,8 +171,6 @@ describe("PreviewManager", () => { const detach = vi.fn(() => { attached = false; }); - const focus = vi.fn(); - const restoreFocus = vi.fn(); const capturePage = vi.fn(async () => image); const sendCommand = vi.fn(async (method: string): Promise => { if (method === "Runtime.evaluate") { @@ -214,7 +212,6 @@ describe("PreviewManager", () => { send: webviewSend, navigationHistory: { canGoBack: () => false, canGoForward: () => false }, setWindowOpenHandler: vi.fn(), - focus, capturePage, debugger: { isAttached: () => attached, @@ -225,12 +222,6 @@ describe("PreviewManager", () => { off: vi.fn(), }, } as never); - getFocusedWebContents.mockReturnValue({ - id: 7, - isDestroyed: () => false, - focus: restoreFocus, - } as never); - yield* manager.createTab("tab_snapshot"); yield* manager.registerWebview("tab_snapshot", 42); yield* Effect.yieldNow; @@ -256,7 +247,6 @@ describe("PreviewManager", () => { captureBeyondViewport: false, }); expect(sendCommand).not.toHaveBeenCalledWith("Page.bringToFront", undefined); - expect(focus).not.toHaveBeenCalled(); const backgroundCdpCaptured = yield* manager.automationSnapshot("tab_snapshot", true); expect(backgroundCdpCaptured.screenshot).toMatchObject({ width: 640, height: 360 }); @@ -265,9 +255,7 @@ describe("PreviewManager", () => { fromSurface: true, captureBeyondViewport: false, }); - expect(sendCommand).toHaveBeenCalledWith("Page.bringToFront", undefined); - expect(focus).toHaveBeenCalledOnce(); - expect(restoreFocus).toHaveBeenCalledOnce(); + expect(sendCommand).not.toHaveBeenCalledWith("Page.bringToFront", undefined); expect(capturePage).not.toHaveBeenCalled(); captureMode = "failure"; @@ -276,10 +264,8 @@ describe("PreviewManager", () => { true, ); expect(backgroundFallbackCaptured.screenshot).toMatchObject({ width: 640, height: 360 }); - expect(focus).toHaveBeenCalledTimes(2); - expect(restoreFocus).toHaveBeenCalledTimes(2); expect(capturePage).toHaveBeenCalledOnce(); - expect(capturePage).toHaveBeenCalledWith(undefined, { stayHidden: false }); + expect(capturePage).toHaveBeenCalledWith(undefined, { stayHidden: true }); capturePage.mockClear(); const foregroundFallbackCaptured = yield* manager.automationSnapshot("tab_snapshot"); @@ -288,6 +274,7 @@ describe("PreviewManager", () => { height: 360, }); expect(capturePage).toHaveBeenCalledOnce(); + expect(capturePage).toHaveBeenCalledWith(undefined, { stayHidden: false }); capturePage.mockClear(); capturePage.mockResolvedValueOnce({ ...image, isEmpty: () => true }); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index da4a851dde5..ff01f12d59b 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2067,6 +2067,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const captureBackgroundPage = Effect.fn("PreviewManager.captureBackgroundPage")(function* ( tabId: string, wc: Electron.WebContents, + background: boolean, ) { const sourceImage = yield* attemptPromise( { @@ -2074,7 +2075,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId, webContentsId: wc.id, }, - () => wc.capturePage(undefined, { stayHidden: false }), + () => wc.capturePage(undefined, { stayHidden: background }), ); return yield* encodeAutomationScreenshot( tabId, @@ -2177,7 +2178,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId, timeoutMs: cdpScreenshotTimeoutMs, }); - const backgroundScreenshotResult = yield* captureBackgroundPage(tabId, wc).pipe( + const backgroundScreenshotResult = yield* captureBackgroundPage(tabId, wc, background).pipe( Effect.timeoutOption(AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS), Effect.exit, ); @@ -2229,39 +2230,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return snapshot; } - const previouslyFocused = yield* attempt( - { operation: "automationSnapshot.getFocusedWebContents", tabId, webContentsId: wc.id }, - () => webContents.getFocusedWebContents(), - ).pipe(Effect.orElseSucceed(() => null)); - const restoreFocus = - previouslyFocused && previouslyFocused.id !== wc.id - ? attempt( - { - operation: "automationSnapshot.restoreFocusedWebContents", - tabId, - webContentsId: previouslyFocused.id, - }, - () => { - if (!previouslyFocused.isDestroyed()) previouslyFocused.focus(); - }, - ).pipe(Effect.ignore) - : Effect.void; - - // A mounted-but-unselected remains a live guest, but Chromium - // does not expose its composited pixels until that guest is foregrounded. - // The renderer stages it transparently while this short capture lease - // activates the guest itself. Restoring the prior WebContents preserves - // both application focus and the user-visible preview selection. + // The renderer briefly stages a non-selected guest so Chromium can expose + // its composited pixels. Do not focus the guest or send Page.bringToFront: + // Electron can otherwise promote the native guest surface above the host + // UI while ignoring the staging wrapper's opacity. const result = yield* withControlSession(tabId, wc, "snapshot", (send) => - Effect.gen(function* () { - yield* attempt( - { operation: "automationSnapshot.focusWebContents", tabId, webContentsId: wc.id }, - () => wc.focus(), - ); - yield* send("Page.bringToFront"); - return yield* captureAutomationSnapshot(tabId, wc, send, true); - }), - ).pipe(Effect.ensuring(restoreFocus)); + captureAutomationSnapshot(tabId, wc, send, true), + ); if (result.detachAfterCapture) yield* detachControlSession(wc.id); const { detachAfterCapture: _, ...snapshot } = result; return snapshot; diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 5dfb7001a4d..4bad1c565ab 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -490,19 +490,12 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) }, ); await ready.bridge.navigate(ready.tabId, resolution.resolvedUrl); - await withPreviewAutomationBackgroundPresentation( + await waitForNavigationReadiness( threadRef, request.requestId, ready.tabId, - remainingOperationBudget(), - async () => - await waitForNavigationReadiness( - threadRef, - request.requestId, - ready.tabId, - input.readiness ?? "load", - remainingOperationBudget(input.timeoutMs ?? request.timeoutMs), - ), + input.readiness ?? "load", + remainingOperationBudget(input.timeoutMs ?? request.timeoutMs), ); return await currentStatus(threadRef, ready.tabId); } @@ -541,13 +534,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) case "setColorScheme": { const ready = await requireReadyTab(); const input = request.input as PreviewAutomationSetColorSchemeInput; - await withPreviewAutomationBackgroundPresentation( - threadRef, - request.requestId, - ready.tabId, - remainingOperationBudget(), - async () => await ready.bridge.setColorScheme(ready.tabId, input.colorScheme), - ); + await ready.bridge.setColorScheme(ready.tabId, input.colorScheme); return { tabId: ready.tabId, colorScheme: input.colorScheme, @@ -565,86 +552,44 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "click": { const ready = await requireReadyTab(); - return await withPreviewAutomationBackgroundPresentation( - threadRef, - request.requestId, + return await ready.bridge.automation.click( ready.tabId, - remainingOperationBudget(), - async () => - await ready.bridge.automation.click( - ready.tabId, - request.input as Parameters[1], - ), + request.input as Parameters[1], ); } case "type": { const ready = await requireReadyTab(); - return await withPreviewAutomationBackgroundPresentation( - threadRef, - request.requestId, + return await ready.bridge.automation.type( ready.tabId, - remainingOperationBudget(), - async () => - await ready.bridge.automation.type( - ready.tabId, - request.input as Parameters[1], - ), + request.input as Parameters[1], ); } case "press": { const ready = await requireReadyTab(); - return await withPreviewAutomationBackgroundPresentation( - threadRef, - request.requestId, + return await ready.bridge.automation.press( ready.tabId, - remainingOperationBudget(), - async () => - await ready.bridge.automation.press( - ready.tabId, - request.input as Parameters[1], - ), + request.input as Parameters[1], ); } case "scroll": { const ready = await requireReadyTab(); - return await withPreviewAutomationBackgroundPresentation( - threadRef, - request.requestId, + return await ready.bridge.automation.scroll( ready.tabId, - remainingOperationBudget(), - async () => - await ready.bridge.automation.scroll( - ready.tabId, - request.input as Parameters[1], - ), + request.input as Parameters[1], ); } case "evaluate": { const ready = await requireReadyTab(); - return await withPreviewAutomationBackgroundPresentation( - threadRef, - request.requestId, + return await ready.bridge.automation.evaluate( ready.tabId, - remainingOperationBudget(), - async () => - await ready.bridge.automation.evaluate( - ready.tabId, - request.input as Parameters[1], - ), + request.input as Parameters[1], ); } case "waitFor": { const ready = await requireReadyTab(); - return await withPreviewAutomationBackgroundPresentation( - threadRef, - request.requestId, + return await ready.bridge.automation.waitFor( ready.tabId, - remainingOperationBudget(), - async () => - await ready.bridge.automation.waitFor( - ready.tabId, - request.input as Parameters[1], - ), + request.input as Parameters[1], ); } case "recordingStart": { From 27d081832b3dcbb04f9ba5dba6b0412641d9bbda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sun, 26 Jul 2026 15:22:39 +0100 Subject: [PATCH 11/43] Harden preview automation deadline handling - Clamp background capture staging to current renderer bounds - Share host deadlines and pass snapshot budgets through desktop IPC - Retain presentation leases until timed-out captures settle - Add focused regression coverage for all review findings --- BRANCH_DETAILS.md | 1 + apps/desktop/src/ipc/methods/preview.ts | 8 +++-- apps/desktop/src/preload.ts | 3 +- apps/desktop/src/preview/Manager.test.ts | 16 +++++++++ apps/desktop/src/preview/Manager.ts | 18 +++++++--- .../src/browser/browserSurfaceStore.test.ts | 23 ++++++++++++ apps/web/src/browser/browserSurfaceStore.ts | 15 ++++++-- .../preview/PreviewAutomationHosts.tsx | 36 ++++++++++++------- .../previewAutomationPresentation.test.ts | 15 ++++++-- .../preview/previewAutomationPresentation.ts | 23 ++++++------ .../previewAutomationRequestConsumer.test.ts | 7 ++++ .../previewAutomationRequestConsumer.ts | 15 ++++---- packages/contracts/src/ipc.test.ts | 17 +++++++-- packages/contracts/src/ipc.ts | 9 ++++- 14 files changed, 163 insertions(+), 43 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index fbe31efa63d..eba3c6edfcc 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -9,6 +9,7 @@ Expected behavior: - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests no longer create their CDP debugger session eagerly when a webview registers. Session initialization is lazy and included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. Closing detached DevTools restores an explicit color-scheme override through the separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. - Inactive preview webviews remain mounted, retain their declared viewport, and stay CSS-visible while positioned outside the human-visible panel. This preserves their runtime and semantic or input automation without selecting them. Background snapshot capture uses a reference-counted presentation lease and always restores the offscreen position afterward; navigation, color-scheme changes, evaluation, waits, and input operations remain offscreen and do not acquire that native-surface lease. Snapshot staging does not change the right-panel tab selected by the user. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: capture staging falls back to a deterministic rectangle fitted inside the renderer viewport. +- A timed-out background snapshot retains its presentation lease until the already-started desktop capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to show the browser use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts report whether the right panel was open, which surface was active, whether the requested browser surface was registered, and whether it had a presentation rectangle. - A newly created preview tab applies its server snapshot and assigned tab id, initiates any requested selection, and acknowledges server-side creation immediately without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching right-panel surface, then waits for stable panel presentation; while the request remains pending it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 3483d5a5339..350090eae65 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -269,9 +269,13 @@ export const automationSnapshot = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, payload: DesktopPreviewAutomationSnapshotInputSchema, result: PreviewAutomationSnapshot, - handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId, background }) { + handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ + tabId, + background, + timeoutMs, + }) { const manager = yield* PreviewManager.PreviewManager; - return yield* manager.automationSnapshot(tabId, background); + return yield* manager.automationSnapshot(tabId, background, timeoutMs); }), }); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index c7b76ead74e..718450e8688 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -203,10 +203,11 @@ contextBridge.exposeInMainWorld("desktopBridge", { automation: { status: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_STATUS_CHANNEL, { tabId }), - snapshot: (tabId, background = false) => + snapshot: (tabId, background = false, timeoutMs) => ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, { tabId, background, + timeoutMs, }), click: (tabId, input) => ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_CLICK_CHANNEL, { tabId, input }), diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index e76c0089202..0684c566aa6 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -300,6 +300,22 @@ describe("PreviewManager", () => { const recovered = yield* manager.automationSnapshot("tab_snapshot"); expect(recovered.screenshot).toMatchObject({ width: 640, height: 360 }); expect(attach).toHaveBeenCalledTimes(2); + + captureMode = "timeout"; + const callerBoundCapture = yield* manager + .automationSnapshot("tab_snapshot", false, 1_000) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(750); + const callerBoundExit = yield* Effect.exit(Fiber.join(callerBoundCapture)); + expect(Exit.isFailure(callerBoundExit)).toBe(true); + if (Exit.isFailure(callerBoundExit)) { + expect(Option.getOrThrow(Cause.findErrorOption(callerBoundExit.cause))).toMatchObject({ + _tag: "PreviewAutomationTimeoutError", + tabId: "tab_snapshot", + timeoutMs: 1_000, + }); + } + expect(detach).toHaveBeenCalledTimes(2); }), ), ); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index ff01f12d59b..1d7071ae0b2 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2219,11 +2219,16 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const automationSnapshot = Effect.fn("PreviewManager.automationSnapshot")(function* ( tabId: string, background = false, + timeoutMs = DEFAULT_AUTOMATION_TIMEOUT_MS, ) { const wc = yield* requireWebContents(tabId); if (!background) { - const result = yield* withControlSession(tabId, wc, "snapshot", (send) => - captureAutomationSnapshot(tabId, wc, send, false), + const result = yield* withControlSession( + tabId, + wc, + "snapshot", + (send) => captureAutomationSnapshot(tabId, wc, send, false), + timeoutMs, ); if (result.detachAfterCapture) yield* detachControlSession(wc.id); const { detachAfterCapture: _, ...snapshot } = result; @@ -2234,8 +2239,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function // its composited pixels. Do not focus the guest or send Page.bringToFront: // Electron can otherwise promote the native guest surface above the host // UI while ignoring the staging wrapper's opacity. - const result = yield* withControlSession(tabId, wc, "snapshot", (send) => - captureAutomationSnapshot(tabId, wc, send, true), + const result = yield* withControlSession( + tabId, + wc, + "snapshot", + (send) => captureAutomationSnapshot(tabId, wc, send, true), + timeoutMs, ); if (result.detachAfterCapture) yield* detachControlSession(wc.id); const { detachAfterCapture: _, ...snapshot } = result; @@ -3159,6 +3168,7 @@ export class PreviewManager extends Context.Service< readonly automationSnapshot: ( tabId: string, background?: boolean, + timeoutMs?: number, ) => Effect.Effect; readonly automationClick: ( tabId: string, diff --git a/apps/web/src/browser/browserSurfaceStore.test.ts b/apps/web/src/browser/browserSurfaceStore.test.ts index ca753a5821e..6e4e08482f5 100644 --- a/apps/web/src/browser/browserSurfaceStore.test.ts +++ b/apps/web/src/browser/browserSurfaceStore.test.ts @@ -94,6 +94,29 @@ describe("browserSurfaceStore", () => { }); }); + it("clamps a stale presented rectangle to the current renderer viewport", () => { + expect( + resolveBrowserSurfaceBackgroundCaptureRect( + { + active: { + rect: { x: 1_000, y: 700, width: 900, height: 640 }, + visible: true, + content: null, + updatedAt: 1, + owner: null, + }, + }, + "hidden", + { width: 800, height: 600 }, + ), + ).toEqual({ + x: 0, + y: 0, + width: 800, + height: 600, + }); + }); + it("ignores updates and releases from a stale surface lease", () => { const tabId = "leased-browser-surface"; const staleRect = { x: 0, y: 0, width: 500, height: 700 }; diff --git a/apps/web/src/browser/browserSurfaceStore.ts b/apps/web/src/browser/browserSurfaceStore.ts index 6f386d48e01..ac191530bf4 100644 --- a/apps/web/src/browser/browserSurfaceStore.ts +++ b/apps/web/src/browser/browserSurfaceStore.ts @@ -71,13 +71,22 @@ export function resolveBrowserSurfaceBackgroundCaptureRect( tabId: string, viewport: { readonly width: number; readonly height: number }, ): BrowserSurfaceRect { - const presentedRect = resolveBrowserSurfacePanelRect(byTabId, tabId); - if (presentedRect) return presentedRect; - const viewportWidth = Number.isFinite(viewport.width) && viewport.width > 0 ? Math.round(viewport.width) : 1280; const viewportHeight = Number.isFinite(viewport.height) && viewport.height > 0 ? Math.round(viewport.height) : 800; + const presentedRect = resolveBrowserSurfacePanelRect(byTabId, tabId); + if (presentedRect) { + const width = Math.max(1, Math.min(Math.round(presentedRect.width), viewportWidth)); + const height = Math.max(1, Math.min(Math.round(presentedRect.height), viewportHeight)); + return { + x: Math.max(0, Math.min(Math.round(presentedRect.x), viewportWidth - width)), + y: Math.max(0, Math.min(Math.round(presentedRect.y), viewportHeight - height)), + width, + height, + }; + } + const width = Math.max(1, Math.min(1280, viewportWidth)); const height = Math.max(1, Math.min(800, viewportHeight)); return { diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 4bad1c565ab..e2f875f42ef 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -52,6 +52,7 @@ import { } from "./previewAutomationPresentation"; import { PreviewAutomationNavigationTimeoutError, + PreviewAutomationHostDeadlineExceededError, PreviewAutomationOperationError, PreviewAutomationOverlayTimeoutError, PreviewAutomationRecordingNotActiveError, @@ -63,6 +64,7 @@ import { resolvePreviewAutomationOpenWaitPolicy } from "./previewAutomationOpenR import { createPreviewAutomationRequestConsumerAtom, previewAutomationExecutionBudget, + previewAutomationRemainingBudget, } from "./previewAutomationRequestConsumer"; import { createPreviewAutomationClientId } from "./previewAutomationClientId"; import { @@ -72,8 +74,6 @@ import { } from "./previewAutomationTarget"; import { isPreviewViewportReady } from "./previewViewportReadiness"; -const PREVIEW_AUTOMATION_INTERNAL_RESPONSE_GRACE_MS = 500; - const waitForDesktopOverlay = async ( threadRef: ScopedThreadRef, requestId: string, @@ -352,14 +352,20 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) const handleRequest = useCallback( async (request: PreviewAutomationRequest): Promise => { - const operationDeadline = - Date.now() + - previewAutomationExecutionBudget( - request.timeoutMs, - PREVIEW_AUTOMATION_INTERNAL_RESPONSE_GRACE_MS, - ); - const remainingOperationBudget = (requestedTimeoutMs = request.timeoutMs): number => - Math.max(1, Math.min(requestedTimeoutMs, operationDeadline - Date.now())); + const operationBudgetMs = previewAutomationExecutionBudget(request.timeoutMs); + const operationDeadline = Date.now() + operationBudgetMs; + const remainingOperationBudget = (requestedTimeoutMs = request.timeoutMs): number => { + const remainingMs = previewAutomationRemainingBudget(operationDeadline, requestedTimeoutMs); + if (remainingMs > 0) return remainingMs; + throw new PreviewAutomationHostDeadlineExceededError({ + requestId: request.requestId, + operation: request.operation, + environmentId, + threadId: request.threadId, + tabId, + timeoutMs: operationBudgetMs, + }); + }; const threadRef: ScopedThreadRef = { environmentId, threadId: request.threadId, @@ -542,12 +548,18 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "snapshot": { const ready = await requireReadyTab(); + const presentationTimeoutMs = remainingOperationBudget(); return await withPreviewAutomationBackgroundPresentation( threadRef, request.requestId, ready.tabId, - remainingOperationBudget(), - async (background) => await ready.bridge.automation.snapshot(ready.tabId, background), + presentationTimeoutMs, + async (background) => + await ready.bridge.automation.snapshot( + ready.tabId, + background, + remainingOperationBudget(), + ), ); } case "click": { diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index ae0a673ff15..9d9eb5b575b 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -119,7 +119,7 @@ describe("preview automation presentation", () => { } }); - it("releases a background capture lease when the staged operation stalls", async () => { + it("retains a background capture lease until a timed-out operation settles", async () => { vi.useFakeTimers(); vi.stubGlobal("document", { querySelectorAll: () => [ @@ -139,12 +139,16 @@ describe("preview automation presentation", () => { }, }); try { + let settleOperation!: () => void; + const stalledOperation = new Promise((resolve) => { + settleOperation = resolve; + }); const operation = withPreviewAutomationBackgroundPresentation( threadRef, "request-stalled", "tab-background", 40, - () => new Promise(() => undefined), + () => stalledOperation, ); const rejection = expect(operation).rejects.toMatchObject({ _tag: "PreviewAutomationBackgroundPresentationTimeoutError", @@ -159,6 +163,13 @@ describe("preview automation presentation", () => { await vi.advanceTimersByTimeAsync(40); await rejection; + expect( + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + ).toBe(1); + + settleOperation(); + await stalledOperation; + await Promise.resolve(); expect( useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], ).toBeUndefined(); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index 9a99b31dafb..01cd635f8ea 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -91,21 +91,24 @@ export async function withPreviewAutomationBackgroundPresentation( }, timeoutMs); }); const operation = (async () => { - await waitForPreviewAutomationBackgroundPresentation({ - threadRef, - requestId, - tabId, - timeoutMs, - }); - if (timedOut) throw timeoutError(); - const stillBackground = !(useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false); - return await use(stillBackground); + try { + await waitForPreviewAutomationBackgroundPresentation({ + threadRef, + requestId, + tabId, + timeoutMs, + }); + if (timedOut) throw timeoutError(); + const stillBackground = !(useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false); + return await use(stillBackground); + } finally { + releaseCapture(); + } })(); try { return await Promise.race([operation, deadline]); } finally { if (timer !== undefined) globalThis.clearTimeout(timer); - releaseCapture(); } } diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts index 9a9a101705b..7e52de78829 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts @@ -20,6 +20,7 @@ import { import { createPreviewAutomationRequestConsumerAtom, previewAutomationExecutionBudget, + previewAutomationRemainingBudget, serializePreviewAutomationError, } from "./previewAutomationRequestConsumer"; @@ -61,6 +62,12 @@ describe("previewAutomationRequestConsumer", () => { expect(previewAutomationExecutionBudget(100, 250)).toBe(100); expect(previewAutomationExecutionBudget(500, 250)).toBe(500); expect(previewAutomationExecutionBudget(1_000, 250)).toBe(750); + expect(previewAutomationExecutionBudget(1_000)).toBe(750); + }); + + it("reports an expired operation budget instead of clamping it to one millisecond", () => { + expect(previewAutomationRemainingBudget(1_000, 15_000, 999)).toBe(1); + expect(previewAutomationRemainingBudget(1_000, 15_000, 1_001)).toBe(-1); }); it("acknowledges a replacement stream before consuming requests from it", async () => { diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts index 92688862b90..77fb6aac8c7 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts @@ -15,13 +15,19 @@ import { type AutomationStreamResult = AsyncResult.AsyncResult; -const PREVIEW_AUTOMATION_RESPONSE_GRACE_MS = 250; +export const PREVIEW_AUTOMATION_RESPONSE_GRACE_MS = 250; export const previewAutomationExecutionBudget = ( timeoutMs: number, - responseGraceMs: number, + responseGraceMs = PREVIEW_AUTOMATION_RESPONSE_GRACE_MS, ): number => (timeoutMs > responseGraceMs * 2 ? timeoutMs - responseGraceMs : timeoutMs); +export const previewAutomationRemainingBudget = ( + operationDeadline: number, + requestedTimeoutMs: number, + now = Date.now(), +): number => Math.min(requestedTimeoutMs, operationDeadline - now); + const handleWithinResponseBudget = ( request: PreviewAutomationRequest, environmentId: PreviewAutomationHost["environmentId"], @@ -29,10 +35,7 @@ const handleWithinResponseBudget = ( ): Promise => new Promise((resolve, reject) => { let settled = false; - const timeoutMs = previewAutomationExecutionBudget( - request.timeoutMs, - PREVIEW_AUTOMATION_RESPONSE_GRACE_MS, - ); + const timeoutMs = previewAutomationExecutionBudget(request.timeoutMs); const timer = globalThis.setTimeout(() => { if (settled) return; settled = true; diff --git a/packages/contracts/src/ipc.test.ts b/packages/contracts/src/ipc.test.ts index 0564adf0fc7..ffc4ece1cca 100644 --- a/packages/contracts/src/ipc.test.ts +++ b/packages/contracts/src/ipc.test.ts @@ -43,11 +43,24 @@ describe("DesktopEnvironmentBootstrapSchema", () => { describe("DesktopPreviewAutomationSnapshotInputSchema", () => { const decode = Schema.decodeUnknownSync(DesktopPreviewAutomationSnapshotInputSchema); - it("defaults omitted and undefined background flags to foreground capture", () => { - expect(decode({ tabId: "tab-1" })).toEqual({ tabId: "tab-1", background: false }); + it("defaults omitted legacy fields to foreground capture and the desktop timeout", () => { + expect(decode({ tabId: "tab-1" })).toEqual({ + tabId: "tab-1", + background: false, + timeoutMs: 15_000, + }); expect(decode({ tabId: "tab-1", background: undefined })).toEqual({ tabId: "tab-1", background: false, + timeoutMs: 15_000, + }); + }); + + it("preserves a caller-supplied snapshot timeout", () => { + expect(decode({ tabId: "tab-1", background: true, timeoutMs: 1_250 })).toEqual({ + tabId: "tab-1", + background: true, + timeoutMs: 1_250, }); }); }); diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index f0792e5344d..be33c1cdfe4 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -913,6 +913,9 @@ export const DesktopPreviewTabInputSchema = Schema.Struct({ export const DesktopPreviewAutomationSnapshotInputSchema = Schema.Struct({ tabId: DesktopPreviewTabIdSchema, background: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + timeoutMs: Schema.Int.check(Schema.isGreaterThan(0)) + .check(Schema.isLessThanOrEqualTo(60_000)) + .pipe(Schema.withDecodingDefault(Effect.succeed(15_000))), }); export const DesktopPreviewRegisterWebviewInputSchema = Schema.Struct({ @@ -1098,7 +1101,11 @@ export interface DesktopPreviewBridge { }; automation: { status: (tabId: string) => Promise; - snapshot: (tabId: string, background: boolean) => Promise; + snapshot: ( + tabId: string, + background: boolean, + timeoutMs?: number, + ) => Promise; click: (tabId: string, input: PreviewAutomationClickInput) => Promise; type: (tabId: string, input: PreviewAutomationTypeInput) => Promise; press: (tabId: string, input: PreviewAutomationPressInput) => Promise; From bfe907cc7442e8cc223cd774661ebbb9a554d4f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Sun, 26 Jul 2026 15:38:53 +0100 Subject: [PATCH 12/43] Fix preview background presentation races - Use selected panel state for background capture - Bound compositor frames and release stalled staging leases - Cover panel-switch and paused-frame regressions --- BRANCH_DETAILS.md | 2 +- .../previewAutomationPresentation.test.ts | 97 +++++++++++++++ .../preview/previewAutomationPresentation.ts | 116 +++++++++++------- 3 files changed, 167 insertions(+), 48 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index eba3c6edfcc..b722935226e 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -9,7 +9,7 @@ Expected behavior: - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests no longer create their CDP debugger session eagerly when a webview registers. Session initialization is lazy and included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. Closing detached DevTools restores an explicit color-scheme override through the separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. - Inactive preview webviews remain mounted, retain their declared viewport, and stay CSS-visible while positioned outside the human-visible panel. This preserves their runtime and semantic or input automation without selecting them. Background snapshot capture uses a reference-counted presentation lease and always restores the offscreen position afterward; navigation, color-scheme changes, evaluation, waits, and input operations remain offscreen and do not acquire that native-surface lease. Snapshot staging does not change the right-panel tab selected by the user. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: capture staging falls back to a deterministic rectangle fitted inside the renderer viewport. -- A timed-out background snapshot retains its presentation lease until the already-started desktop capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. +- A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to show the browser use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts report whether the right panel was open, which surface was active, whether the requested browser surface was registered, and whether it had a presentation rectangle. - A newly created preview tab applies its server snapshot and assigned tab id, initiates any requested selection, and acknowledges server-side creation immediately without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching right-panel surface, then waits for stable panel presentation; while the request remains pending it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index 9d9eb5b575b..f380ea3eadd 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -119,6 +119,103 @@ describe("preview automation presentation", () => { } }); + it("stages a visible surface when another right-panel surface is selected", async () => { + vi.stubGlobal("document", { + querySelectorAll: () => [ + { + dataset: { + previewViewport: "tab-background", + previewBackgroundCapture: "true", + }, + offsetWidth: 800, + }, + ], + }); + vi.stubGlobal("window", { + requestAnimationFrame: (callback: FrameRequestCallback) => { + callback(0); + return 1; + }, + }); + const surface = acquireBrowserSurface("tab-background"); + try { + surface.present({ x: 0, y: 0, width: 800, height: 600 }, true); + revealPreviewAutomationTab(threadRef, "tab-foreground"); + + const background = await withPreviewAutomationBackgroundPresentation( + threadRef, + "request-background", + "tab-background", + 40, + async (isBackground) => { + expect( + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + ).toBe(1); + return isBackground; + }, + ); + + expect(background).toBe(true); + expect( + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + ).toBeUndefined(); + } finally { + surface.release(); + vi.unstubAllGlobals(); + } + }); + + it("releases a background capture lease when compositor frames remain paused", async () => { + vi.useFakeTimers(); + const cancelAnimationFrame = vi.fn(); + vi.stubGlobal("document", { + querySelectorAll: () => [ + { + dataset: { + previewViewport: "tab-background", + previewBackgroundCapture: "true", + }, + offsetWidth: 800, + }, + ], + }); + vi.stubGlobal("window", { + requestAnimationFrame: () => 1, + cancelAnimationFrame, + }); + const use = vi.fn(); + try { + const operation = withPreviewAutomationBackgroundPresentation( + threadRef, + "request-paused-frame", + "tab-background", + 40, + use, + ); + const rejection = expect(operation).rejects.toMatchObject({ + _tag: "PreviewAutomationBackgroundPresentationTimeoutError", + requestId: "request-paused-frame", + tabId: "tab-background", + timeoutMs: 40, + }); + expect( + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + ).toBe(1); + + await vi.advanceTimersByTimeAsync(40); + await rejection; + + expect(use).not.toHaveBeenCalled(); + expect(cancelAnimationFrame).toHaveBeenCalledWith(1); + expect( + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + ).toBeUndefined(); + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); + it("retains a background capture lease until a timed-out operation settles", async () => { vi.useFakeTimers(); vi.stubGlobal("document", { diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index 01cd635f8ea..f8e51569e13 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -9,6 +9,50 @@ import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelSto import { PreviewAutomationBackgroundPresentationTimeoutError } from "./previewAutomationErrors"; +interface PreviewAutomationBackgroundPresentationInput { + readonly threadRef: ScopedThreadRef; + readonly requestId: string; + readonly tabId: string; + readonly timeoutMs: number; +} + +function backgroundPresentationTimeoutError( + input: PreviewAutomationBackgroundPresentationInput, +): PreviewAutomationBackgroundPresentationTimeoutError { + return new PreviewAutomationBackgroundPresentationTimeoutError({ + requestId: input.requestId, + environmentId: input.threadRef.environmentId, + threadId: input.threadRef.threadId, + tabId: input.tabId, + timeoutMs: input.timeoutMs, + }); +} + +async function waitForPreviewAutomationCompositorFrame( + deadline: number, + timeoutError: () => PreviewAutomationBackgroundPresentationTimeoutError, +): Promise { + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) throw timeoutError(); + + await new Promise((resolve, reject) => { + let settled = false; + let animationFrameId: number | undefined; + const timer = globalThis.setTimeout(() => { + if (settled) return; + settled = true; + if (animationFrameId !== undefined) window.cancelAnimationFrame?.(animationFrameId); + reject(timeoutError()); + }, remainingMs); + animationFrameId = window.requestAnimationFrame(() => { + if (settled) return; + settled = true; + globalThis.clearTimeout(timer); + resolve(); + }); + }); +} + export function revealPreviewAutomationTab(ref: ScopedThreadRef, tabId: string): void { setActivePreviewTab(ref, tabId); useRightPanelStore.getState().openBrowser(ref, tabId); @@ -23,13 +67,11 @@ export function isPreviewAutomationTabPresented(ref: ScopedThreadRef, tabId: str ); } -export async function waitForPreviewAutomationBackgroundPresentation(input: { - readonly threadRef: ScopedThreadRef; - readonly requestId: string; - readonly tabId: string; - readonly timeoutMs: number; -}): Promise { +export async function waitForPreviewAutomationBackgroundPresentation( + input: PreviewAutomationBackgroundPresentationInput, +): Promise { const deadline = Date.now() + input.timeoutMs; + const timeoutError = () => backgroundPresentationTimeoutError(input); while (true) { if (isPreviewAutomationTabPresented(input.threadRef, input.tabId)) return; @@ -44,8 +86,8 @@ export async function waitForPreviewAutomationBackgroundPresentation(input: { // Force the staged wrapper through layout, then allow Chromium two // compositor frames before asking the guest WebContents for pixels. void wrapper.offsetWidth; - await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); - await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); + await waitForPreviewAutomationCompositorFrame(deadline, timeoutError); + await waitForPreviewAutomationCompositorFrame(deadline, timeoutError); return; } @@ -54,13 +96,7 @@ export async function waitForPreviewAutomationBackgroundPresentation(input: { await new Promise((resolve) => window.setTimeout(resolve, Math.min(16, remainingMs))); } - throw new PreviewAutomationBackgroundPresentationTimeoutError({ - requestId: input.requestId, - environmentId: input.threadRef.environmentId, - threadId: input.threadRef.threadId, - tabId: input.tabId, - timeoutMs: input.timeoutMs, - }); + throw timeoutError(); } export async function withPreviewAutomationBackgroundPresentation( @@ -70,45 +106,31 @@ export async function withPreviewAutomationBackgroundPresentation( timeoutMs: number, use: (background: boolean) => Promise, ): Promise { - const background = !(useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false); + const background = !isPreviewAutomationTabPresented(threadRef, tabId); if (!background) return await use(false); - const timeoutError = () => - new PreviewAutomationBackgroundPresentationTimeoutError({ - requestId, - environmentId: threadRef.environmentId, - threadId: threadRef.threadId, - tabId, - timeoutMs, - }); + const input = { threadRef, requestId, tabId, timeoutMs }; + const timeoutError = () => backgroundPresentationTimeoutError(input); + const deadline = Date.now() + timeoutMs; const releaseCapture = acquireBrowserSurfaceBackgroundCapture(tabId); - let timedOut = false; + let captureStarted = false; let timer: ReturnType | undefined; - const deadline = new Promise((_resolve, reject) => { - timer = globalThis.setTimeout(() => { - timedOut = true; - reject(timeoutError()); - }, timeoutMs); - }); - const operation = (async () => { - try { - await waitForPreviewAutomationBackgroundPresentation({ - threadRef, - requestId, - tabId, - timeoutMs, - }); - if (timedOut) throw timeoutError(); - const stillBackground = !(useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false); - return await use(stillBackground); - } finally { - releaseCapture(); - } - })(); try { - return await Promise.race([operation, deadline]); + await waitForPreviewAutomationBackgroundPresentation(input); + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) throw timeoutError(); + + const stillBackground = !isPreviewAutomationTabPresented(threadRef, tabId); + const capture = use(stillBackground); + captureStarted = true; + const operation = capture.finally(releaseCapture); + const captureDeadline = new Promise((_resolve, reject) => { + timer = globalThis.setTimeout(() => reject(timeoutError()), remainingMs); + }); + return await Promise.race([operation, captureDeadline]); } finally { if (timer !== undefined) globalThis.clearTimeout(timer); + if (!captureStarted) releaseCapture(); } } From 5ead5188099613633a262cbb7bd788a066232c22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Mon, 27 Jul 2026 08:45:43 +0100 Subject: [PATCH 13/43] Clarify preview worktree port selection --- BRANCH_DETAILS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 51b948a8189..a66865997a9 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -51,3 +51,4 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/preview/Manager.test.ts - Web: `5744` - Server/WebSocket: `13784` +- Start the dev runner with `T3CODE_PORT_OFFSET=11` to select these branch-fixed ports instead of upstream's worktree-path-derived starting offset. The runner can still advance when either port is unavailable, so confirm the printed ports before testing. From b2a4e0baab35ba1dd4212fbe27c930cd02f827f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Mon, 27 Jul 2026 08:47:18 +0100 Subject: [PATCH 14/43] Refresh preview branch documentation --- BRANCH_DETAILS.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index a66865997a9..d23d0b2d7ad 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -1,6 +1,6 @@ # Preview Automation Reliability -Product-native preview automation remains bounded and recoverable across the web host, MCP server, and Electron CDP controller. Preserve this behavior until upstream provides equivalent operation deadlines, control-session recovery, and degraded semantic snapshots. +Product-native preview automation is bounded and recoverable across the web host, MCP server, and Electron CDP controller. The branch-specific layer covers operation deadlines, control-session recovery, background capture presentation, and degraded semantic snapshots. Expected behavior: @@ -13,7 +13,7 @@ Expected behavior: - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to show the browser use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts report whether the right panel was open, which surface was active, whether the requested browser surface was registered, and whether it had a presentation rectangle. - A newly created preview tab applies its server snapshot and assigned tab id, initiates any requested selection, and acknowledges server-side creation immediately without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching right-panel surface, then waits for stable panel presentation; while the request remains pending it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. -- The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes follow upstream's single-origin architecture: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode continues to pin `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. +- The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use single-origin routing: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. Current limitations: @@ -30,6 +30,7 @@ Primary files: - `apps/web/src/browser/HostedBrowserWebview.tsx` - `apps/web/src/browser/browserSurfaceStore.ts` - `apps/web/src/browser/hostedBrowserWebviewStyle.ts` +- `apps/web/src/components/auth/PairingRouteSurface.logic.ts` - `apps/web/src/components/auth/PairingRouteSurface.tsx` - `apps/web/src/components/preview/PreviewAutomationHosts.tsx` - `apps/web/src/components/preview/previewAutomationPresentation.ts` @@ -51,4 +52,4 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/preview/Manager.test.ts - Web: `5744` - Server/WebSocket: `13784` -- Start the dev runner with `T3CODE_PORT_OFFSET=11` to select these branch-fixed ports instead of upstream's worktree-path-derived starting offset. The runner can still advance when either port is unavailable, so confirm the printed ports before testing. +- Start the dev runner with `T3CODE_PORT_OFFSET=11` to select these branch-fixed ports. This overrides the worktree-path-derived starting offset. The runner can still advance when either port is unavailable, so confirm the printed ports before testing. From d6ec942f82b1cd26cac03cb48eb005cbcf74dec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Mon, 27 Jul 2026 08:57:49 +0100 Subject: [PATCH 15/43] Clarify preview branch dev guidance - Preserve upstream attribution for single-origin browser dev - Document optional preferred ports and collision handling --- BRANCH_DETAILS.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index d23d0b2d7ad..0c3b511f604 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -13,7 +13,7 @@ Expected behavior: - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to show the browser use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts report whether the right panel was open, which surface was active, whether the requested browser surface was registered, and whether it had a presentation rectangle. - A newly created preview tab applies its server snapshot and assigned tab id, initiates any requested selection, and acknowledges server-side creation immediately without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching right-panel surface, then waits for stable panel presentation; while the request remains pending it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. -- The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use single-origin routing: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. +- The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes follow upstream's single-origin architecture: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode continues to pin `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. Current limitations: @@ -50,6 +50,9 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/preview/Manager.test.ts ## Development Ports +Preferred ports when explicitly selecting offset 11: + - Web: `5744` - Server/WebSocket: `13784` -- Start the dev runner with `T3CODE_PORT_OFFSET=11` to select these branch-fixed ports. This overrides the worktree-path-derived starting offset. The runner can still advance when either port is unavailable, so confirm the printed ports before testing. +- This explicit offset is optional. Use `T3CODE_PORT_OFFSET=11 vp run dev` only when you need to prefer this documented pair instead of the worktree-path-derived starting offset. +- If either preferred port is unavailable, the runner can advance. Use the `serverPort` and `webPort` values from the printed `[dev-runner]` line for testing; if a test requires the documented pair exactly, free the conflicting ports and restart. From 80f4cc3b258d9cf7bcf1ce1547e3d1aa009d8f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Mon, 27 Jul 2026 12:04:21 +0100 Subject: [PATCH 16/43] Fix preview snapshot capture and browser surface rendering --- apps/desktop/src/preview/Manager.test.ts | 71 +++++++- apps/desktop/src/preview/Manager.ts | 158 +++++++++++++++--- apps/web/src/browser/HostedBrowserWebview.tsx | 28 ++-- .../src/browser/browserSurfaceStore.test.ts | 18 ++ apps/web/src/browser/browserSurfaceStore.ts | 15 ++ .../previewAutomationPresentation.test.ts | 20 +-- .../preview/previewAutomationPresentation.ts | 22 ++- 7 files changed, 271 insertions(+), 61 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 0684c566aa6..d0835ef9699 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -20,6 +20,11 @@ import * as BrowserSession from "./BrowserSession.ts"; import * as PreviewManager from "./Manager.ts"; const { + BrowserWindow, + bridgeAttach, + bridgeDestroy, + bridgeIsDestroyed, + bridgeSendCommand, createFromBuffer, createFromPath, fromId, @@ -30,6 +35,22 @@ const { writeFile, writeImage, } = vi.hoisted(() => ({ + BrowserWindow: vi.fn(function () { + return { + isDestroyed: bridgeIsDestroyed, + destroy: bridgeDestroy, + webContents: { + debugger: { + attach: bridgeAttach, + sendCommand: bridgeSendCommand, + }, + }, + }; + }), + bridgeAttach: vi.fn(), + bridgeDestroy: vi.fn(), + bridgeIsDestroyed: vi.fn(() => false), + bridgeSendCommand: vi.fn(), createFromBuffer: vi.fn(), createFromPath: vi.fn((): { readonly isEmpty: () => boolean } => ({ isEmpty: () => false })), fromId: vi.fn(() => null), @@ -42,6 +63,7 @@ const { })); vi.mock("electron", () => ({ + BrowserWindow, clipboard: { writeImage, }, @@ -111,6 +133,12 @@ const withManager = ( describe("PreviewManager", () => { beforeEach(() => { + BrowserWindow.mockClear(); + bridgeAttach.mockClear(); + bridgeDestroy.mockClear(); + bridgeIsDestroyed.mockReset(); + bridgeIsDestroyed.mockReturnValue(false); + bridgeSendCommand.mockReset(); fromId.mockClear(); getFocusedWebContents.mockReset(); getFocusedWebContents.mockReturnValue(null); @@ -189,6 +217,9 @@ describe("PreviewManager", () => { if (method === "Accessibility.getFullAXTree") { return { nodes: [] }; } + if (method === "Target.getTargetInfo") { + return { targetInfo: { targetId: "target-42" } }; + } if (method === "Page.captureScreenshot") { if (captureMode === "failure") throw new Error("UnknownVizError"); if (captureMode === "timeout") return await new Promise(() => undefined); @@ -196,6 +227,20 @@ describe("PreviewManager", () => { } return undefined; }); + bridgeSendCommand.mockImplementation( + async (method: string, _params?: unknown, sessionId?: string): Promise => { + if (method === "Target.attachToTarget") { + return { sessionId: "target-session-42" }; + } + if (method === "Page.captureScreenshot") { + expect(sessionId).toBe("target-session-42"); + if (captureMode === "failure") throw new Error("UnknownVizError"); + if (captureMode === "timeout") return await new Promise(() => undefined); + return { data: png.toString("base64") }; + } + return {}; + }, + ); fromId.mockReturnValue({ id: 42, isDestroyed: () => false, @@ -250,11 +295,28 @@ describe("PreviewManager", () => { const backgroundCdpCaptured = yield* manager.automationSnapshot("tab_snapshot", true); expect(backgroundCdpCaptured.screenshot).toMatchObject({ width: 640, height: 360 }); - expect(sendCommand).toHaveBeenCalledWith("Page.captureScreenshot", { - format: "png", - fromSurface: true, - captureBeyondViewport: false, + expect(BrowserWindow).toHaveBeenCalledWith({ + show: false, + webPreferences: { sandbox: true }, + }); + expect(bridgeAttach).toHaveBeenCalledWith("1.3"); + expect(bridgeSendCommand).toHaveBeenCalledWith("Target.attachToTarget", { + targetId: "target-42", + flatten: true, + }); + expect(bridgeSendCommand).toHaveBeenCalledWith( + "Page.captureScreenshot", + { + format: "png", + fromSurface: true, + captureBeyondViewport: false, + }, + "target-session-42", + ); + expect(bridgeSendCommand).toHaveBeenCalledWith("Target.detachFromTarget", { + sessionId: "target-session-42", }); + expect(bridgeDestroy).toHaveBeenCalledOnce(); expect(sendCommand).not.toHaveBeenCalledWith("Page.bringToFront", undefined); expect(capturePage).not.toHaveBeenCalled(); @@ -266,6 +328,7 @@ describe("PreviewManager", () => { expect(backgroundFallbackCaptured.screenshot).toMatchObject({ width: 640, height: 360 }); expect(capturePage).toHaveBeenCalledOnce(); expect(capturePage).toHaveBeenCalledWith(undefined, { stayHidden: true }); + expect(bridgeDestroy).toHaveBeenCalledTimes(2); capturePage.mockClear(); const foregroundFallbackCaptured = yield* manager.automationSnapshot("tab_snapshot"); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 1d7071ae0b2..8b06e3ba57e 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -28,14 +28,7 @@ import type { } from "@t3tools/contracts"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { normalizePreviewUrl } from "@t3tools/shared/preview"; -import { - type BrowserWindow, - type Session, - clipboard, - nativeImage, - shell, - webContents, -} from "electron"; +import { BrowserWindow, type Session, clipboard, nativeImage, shell, webContents } from "electron"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; @@ -104,7 +97,7 @@ const MAX_SCREENSHOT_WIDTH = 1280; const DEFAULT_AUTOMATION_TIMEOUT_MS = 15_000; const AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS = 250; const AUTOMATION_SCREENSHOT_TIMEOUT_MS = 5_000; -const AUTOMATION_BACKGROUND_CDP_SCREENSHOT_TIMEOUT_MS = 2_000; +const AUTOMATION_BACKGROUND_TARGET_SCREENSHOT_TIMEOUT_MS = 2_000; const AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS = 3_000; const automationExecutionBudget = (timeoutMs: number): number => timeoutMs > AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS * 2 @@ -2064,6 +2057,126 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }, ); + const captureAutomationTargetScreenshot = Effect.fn( + "PreviewManager.captureAutomationTargetScreenshot", + )(function* (tabId: string, wc: Electron.WebContents, send: SendCommand) { + // Electron's debugger session for a staged can keep serving + // Runtime and Accessibility commands while Page.captureScreenshot never + // resolves. Attach a short-lived debugger hosted by an invisible window + // to the exact guest target instead. This uses Electron's in-process CDP + // API and does not expose a remote debugging port. + const targetInfoResponse = yield* send("Target.getTargetInfo"); + const targetId = + typeof targetInfoResponse === "object" && + targetInfoResponse !== null && + "targetInfo" in targetInfoResponse && + typeof targetInfoResponse.targetInfo === "object" && + targetInfoResponse.targetInfo !== null && + "targetId" in targetInfoResponse.targetInfo && + typeof targetInfoResponse.targetInfo.targetId === "string" + ? targetInfoResponse.targetInfo.targetId + : null; + if (targetId === null) { + return yield* new PreviewOperationError({ + operation: "automationSnapshot.resolveTarget", + tabId, + webContentsId: wc.id, + cause: new Error("Target.getTargetInfo returned no target id"), + }); + } + + const bridgeWindow = yield* attempt( + { + operation: "automationSnapshot.createTargetBridge", + tabId, + webContentsId: wc.id, + }, + () => + new BrowserWindow({ + show: false, + webPreferences: { + sandbox: true, + }, + }), + ); + const response = yield* attemptPromise( + { + operation: "automationSnapshot.captureTarget", + tabId, + webContentsId: wc.id, + }, + async () => { + const bridgeDebugger = bridgeWindow.webContents.debugger; + bridgeDebugger.attach("1.3"); + const attached = (await bridgeDebugger.sendCommand("Target.attachToTarget", { + targetId, + flatten: true, + })) as { sessionId?: unknown }; + if (typeof attached.sessionId !== "string") { + throw new Error("Target.attachToTarget returned no session id"); + } + try { + return await bridgeDebugger.sendCommand( + "Page.captureScreenshot", + { + format: "png", + fromSurface: true, + captureBeyondViewport: false, + }, + attached.sessionId, + ); + } finally { + await bridgeDebugger + .sendCommand("Target.detachFromTarget", { sessionId: attached.sessionId }) + .catch(() => undefined); + } + }, + ).pipe( + Effect.ensuring( + attempt( + { + operation: "automationSnapshot.destroyTargetBridge", + tabId, + webContentsId: wc.id, + }, + () => { + if (!bridgeWindow.isDestroyed()) bridgeWindow.destroy(); + }, + ).pipe(Effect.ignore), + ), + ); + const data = + typeof response === "object" && + response !== null && + "data" in response && + typeof response.data === "string" && + response.data.length > 0 + ? response.data + : null; + if (data === null) { + return yield* new PreviewOperationError({ + operation: "automationSnapshot.decodeTargetScreenshot", + tabId, + webContentsId: wc.id, + cause: new Error("Target Page.captureScreenshot returned no PNG data"), + }); + } + const sourceImage = yield* attempt( + { + operation: "automationSnapshot.createTargetImage", + tabId, + webContentsId: wc.id, + }, + () => nativeImage.createFromBuffer(Buffer.from(data, "base64")), + ); + return yield* encodeAutomationScreenshot( + tabId, + wc, + sourceImage, + "automationSnapshot.createTargetImage", + ); + }); + const captureBackgroundPage = Effect.fn("PreviewManager.captureBackgroundPage")(function* ( tabId: string, wc: Electron.WebContents, @@ -2158,25 +2271,26 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function Ref.get(diagnosticsRef), Ref.get(actionTimelineRef), ]); - const cdpScreenshotTimeoutMs = background - ? AUTOMATION_BACKGROUND_CDP_SCREENSHOT_TIMEOUT_MS + const primaryScreenshotTimeoutMs = background + ? AUTOMATION_BACKGROUND_TARGET_SCREENSHOT_TIMEOUT_MS : AUTOMATION_SCREENSHOT_TIMEOUT_MS; - const cdpScreenshotResult = yield* captureAutomationScreenshot(tabId, wc, send).pipe( - Effect.timeoutOption(cdpScreenshotTimeoutMs), - Effect.exit, - ); + const primaryScreenshotResult = yield* ( + background + ? captureAutomationTargetScreenshot(tabId, wc, send) + : captureAutomationScreenshot(tabId, wc, send) + ).pipe(Effect.timeoutOption(primaryScreenshotTimeoutMs), Effect.exit); let screenshot: PreviewAutomationSnapshot["screenshot"] = - Exit.isSuccess(cdpScreenshotResult) && Option.isSome(cdpScreenshotResult.value) - ? cdpScreenshotResult.value.value + Exit.isSuccess(primaryScreenshotResult) && Option.isSome(primaryScreenshotResult.value) + ? primaryScreenshotResult.value.value : null; const detachAfterCapture = - Exit.isSuccess(cdpScreenshotResult) && Option.isNone(cdpScreenshotResult.value); + Exit.isSuccess(primaryScreenshotResult) && Option.isNone(primaryScreenshotResult.value); if (screenshot === null) { - const cdpFailure = Exit.isFailure(cdpScreenshotResult) - ? cdpScreenshotResult.cause + const primaryFailure = Exit.isFailure(primaryScreenshotResult) + ? primaryScreenshotResult.cause : new PreviewAutomationTimeoutError({ tabId, - timeoutMs: cdpScreenshotTimeoutMs, + timeoutMs: primaryScreenshotTimeoutMs, }); const backgroundScreenshotResult = yield* captureBackgroundPage(tabId, wc, background).pipe( Effect.timeoutOption(AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS), @@ -2198,7 +2312,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId, webContentsId: wc.id, background, - cdpCause: cdpFailure, + primaryCause: primaryFailure, backgroundCause: backgroundFailure, }); } diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index 8bff460417d..cdfc47eb8ea 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -12,6 +12,7 @@ import { stopBrowserRecording, useActiveBrowserRecordingTabId } from "./browserR import { resolveBrowserSurfaceBackgroundCaptureRect, resolveBrowserSurfacePanelRect, + selectBrowserSurfaceRenderState, useBrowserSurfaceStore, } from "./browserSurfaceStore"; import { browserViewportSettingKey } from "./browserViewportLayout"; @@ -52,22 +53,19 @@ export function HostedBrowserWebview(props: { const webviewRef = useRef(null); const [aspectRatioLocked, setAspectRatioLocked] = useState(false); const activeRecordingTabId = useActiveBrowserRecordingTabId(); - const presentation = useBrowserSurfaceStore( - useShallow((state) => { - const current = state.byTabId[tabId]; - const backgroundCapture = (state.backgroundCaptureCountByTabId[tabId] ?? 0) > 0; - return { - backgroundCapture, - rect: backgroundCapture - ? resolveBrowserSurfaceBackgroundCaptureRect(state.byTabId, tabId, { - width: window.innerWidth, - height: window.innerHeight, - }) - : resolveBrowserSurfacePanelRect(state.byTabId, tabId), - visible: current?.visible ?? false, - }; - }), + const surface = useBrowserSurfaceStore( + useShallow((state) => selectBrowserSurfaceRenderState(state, tabId)), ); + const presentation = { + backgroundCapture: surface.backgroundCapture, + rect: surface.backgroundCapture + ? resolveBrowserSurfaceBackgroundCaptureRect(surface.byTabId, tabId, { + width: window.innerWidth, + height: window.innerHeight, + }) + : resolveBrowserSurfacePanelRect(surface.byTabId, tabId), + visible: surface.visible, + }; usePreviewBridge({ threadRef, tabId }); useEffect(() => { diff --git a/apps/web/src/browser/browserSurfaceStore.test.ts b/apps/web/src/browser/browserSurfaceStore.test.ts index 6e4e08482f5..6f11cbb95c9 100644 --- a/apps/web/src/browser/browserSurfaceStore.test.ts +++ b/apps/web/src/browser/browserSurfaceStore.test.ts @@ -1,10 +1,12 @@ import { beforeEach, describe, expect, it } from "vite-plus/test"; +import { shallow } from "zustand/shallow"; import { acquireBrowserSurface, acquireBrowserSurfaceBackgroundCapture, resolveBrowserSurfaceBackgroundCaptureRect, resolveBrowserSurfacePanelRect, + selectBrowserSurfaceRenderState, useBrowserSurfaceStore, } from "./browserSurfaceStore"; @@ -33,6 +35,22 @@ describe("browserSurfaceStore", () => { ).toBeUndefined(); }); + it("selects stable inputs while a background capture rect is derived", () => { + const release = acquireBrowserSurfaceBackgroundCapture("tab-background"); + const state = useBrowserSurfaceStore.getState(); + const first = selectBrowserSurfaceRenderState(state, "tab-background"); + const second = selectBrowserSurfaceRenderState(state, "tab-background"); + + expect(shallow(first, second)).toBe(true); + expect(first).toEqual({ + byTabId: state.byTabId, + backgroundCapture: true, + visible: false, + }); + expect(first).not.toHaveProperty("rect"); + release(); + }); + it("tracks content dimensions for a browser that has never been visible", () => { const tabId = "hidden-browser-surface-content-test"; useBrowserSurfaceStore.getState().presentContent(tabId, { diff --git a/apps/web/src/browser/browserSurfaceStore.ts b/apps/web/src/browser/browserSurfaceStore.ts index ac191530bf4..0dfc595e615 100644 --- a/apps/web/src/browser/browserSurfaceStore.ts +++ b/apps/web/src/browser/browserSurfaceStore.ts @@ -46,6 +46,21 @@ export interface BrowserSurfaceLease { readonly release: () => void; } +export function selectBrowserSurfaceRenderState( + state: { + readonly byTabId: Record; + readonly backgroundCaptureCountByTabId: Record; + }, + tabId: string, +) { + const current = state.byTabId[tabId]; + return { + byTabId: state.byTabId, + backgroundCapture: (state.backgroundCaptureCountByTabId[tabId] ?? 0) > 0, + visible: current?.visible ?? false, + }; +} + export function resolveBrowserSurfacePanelRect( byTabId: Readonly>, tabId: string, diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index f380ea3eadd..4b68cbda072 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -165,7 +165,7 @@ describe("preview automation presentation", () => { } }); - it("releases a background capture lease when compositor frames remain paused", async () => { + it("falls back to frame timers when compositor animation frames remain paused", async () => { vi.useFakeTimers(); const cancelAnimationFrame = vi.fn(); vi.stubGlobal("document", { @@ -183,7 +183,7 @@ describe("preview automation presentation", () => { requestAnimationFrame: () => 1, cancelAnimationFrame, }); - const use = vi.fn(); + const use = vi.fn(async (background: boolean) => background); try { const operation = withPreviewAutomationBackgroundPresentation( threadRef, @@ -192,21 +192,17 @@ describe("preview automation presentation", () => { 40, use, ); - const rejection = expect(operation).rejects.toMatchObject({ - _tag: "PreviewAutomationBackgroundPresentationTimeoutError", - requestId: "request-paused-frame", - tabId: "tab-background", - timeoutMs: 40, - }); expect( useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], ).toBe(1); - await vi.advanceTimersByTimeAsync(40); - await rejection; + await vi.advanceTimersByTimeAsync(32); + await expect(operation).resolves.toBe(true); - expect(use).not.toHaveBeenCalled(); - expect(cancelAnimationFrame).toHaveBeenCalledWith(1); + expect(use).toHaveBeenCalledWith(true); + expect(cancelAnimationFrame).toHaveBeenCalledTimes(2); + expect(cancelAnimationFrame).toHaveBeenNthCalledWith(1, 1); + expect(cancelAnimationFrame).toHaveBeenNthCalledWith(2, 1); expect( useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], ).toBeUndefined(); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index f8e51569e13..2eaa9661912 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -16,6 +16,8 @@ interface PreviewAutomationBackgroundPresentationInput { readonly timeoutMs: number; } +const PREVIEW_AUTOMATION_COMPOSITOR_FRAME_FALLBACK_MS = 16; + function backgroundPresentationTimeoutError( input: PreviewAutomationBackgroundPresentationInput, ): PreviewAutomationBackgroundPresentationTimeoutError { @@ -35,21 +37,23 @@ async function waitForPreviewAutomationCompositorFrame( const remainingMs = deadline - Date.now(); if (remainingMs <= 0) throw timeoutError(); - await new Promise((resolve, reject) => { + await new Promise((resolve) => { let settled = false; let animationFrameId: number | undefined; - const timer = globalThis.setTimeout(() => { + const complete = () => { if (settled) return; settled = true; if (animationFrameId !== undefined) window.cancelAnimationFrame?.(animationFrameId); - reject(timeoutError()); - }, remainingMs); - animationFrameId = window.requestAnimationFrame(() => { - if (settled) return; - settled = true; globalThis.clearTimeout(timer); resolve(); - }); + }; + const timer = globalThis.setTimeout( + () => { + complete(); + }, + Math.min(PREVIEW_AUTOMATION_COMPOSITOR_FRAME_FALLBACK_MS, remainingMs), + ); + animationFrameId = window.requestAnimationFrame(complete); }); } @@ -85,6 +89,8 @@ export async function waitForPreviewAutomationBackgroundPresentation( if (wrapper) { // Force the staged wrapper through layout, then allow Chromium two // compositor frames before asking the guest WebContents for pixels. + // Electron can pause the host renderer's animation frames after placing + // a native guest over it, so each wait falls back to one frame interval. void wrapper.offsetWidth; await waitForPreviewAutomationCompositorFrame(deadline, timeoutError); await waitForPreviewAutomationCompositorFrame(deadline, timeoutError); From a87a922ecb2eae7fb15fdfd49d0b7e9900b0a33e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Mon, 27 Jul 2026 15:13:42 +0100 Subject: [PATCH 17/43] Clarify preview presentation timeout diagnostics --- BRANCH_DETAILS.md | 2 +- .../preview/PreviewAutomationHosts.tsx | 14 ++------ .../preview/previewAutomationErrors.ts | 4 +++ .../previewAutomationPresentation.test.ts | 32 +++++++++++++++++++ .../preview/previewAutomationPresentation.ts | 25 +++++++++++++++ .../previewAutomationRequestConsumer.test.ts | 16 +++++++--- 6 files changed, 76 insertions(+), 17 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 42d9a0289e2..7861c689bc0 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -11,7 +11,7 @@ Expected behavior: - Inactive preview webviews remain mounted, retain their declared viewport, and stay CSS-visible while positioned outside the human-visible panel. This preserves their runtime and semantic or input automation without selecting them. Background snapshot capture uses a reference-counted presentation lease and always restores the offscreen position afterward; navigation, color-scheme changes, evaluation, waits, recording, and input operations remain compatible with the selected inline preview or right-panel surface without acquiring that lease. Snapshot staging does not change the human-selected preview surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target in either the inline mini-player or right panel while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: capture staging falls back to a deterministic rectangle fitted inside the renderer viewport. - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. -- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts report the active inline or right-panel surface, whether the requested browser surface was registered, and whether it had a presentation rectangle. +- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface, which presentation kind is active, whether the requested browser surface was registered, and whether it had a presentation rectangle. - A newly created preview tab applies its server snapshot and assigned tab id, initiates any requested selection, and acknowledges server-side creation immediately without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for upstream's `open` input. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes follow upstream's single-origin architecture: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode continues to pin `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 24677ccbf35..aa23b601881 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -29,8 +29,6 @@ import { reconcilePreviewServerSessions, updatePreviewServerSnapshot, } from "~/previewStateStore"; -import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; -import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelStore"; import { resolveBrowserNavigationTarget } from "~/browser/browserTargetResolver"; import { readActiveBrowserRecordingTabIds, @@ -48,6 +46,7 @@ import { useAtomCommand } from "~/state/use-atom-command"; import { previewBridge } from "./previewBridge"; import { isPreviewAutomationTabPresented, + readPreviewAutomationPresentationDiagnostics, revealPreviewAutomationTab, withPreviewAutomationBackgroundPresentation, } from "./previewAutomationPresentation"; @@ -126,22 +125,13 @@ const waitForBrowserSurfaceVisibility = async ( } await new Promise((resolve) => window.setTimeout(resolve, 50)); } - const panel = selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, threadRef); - const miniPlayer = selectThreadPreviewMiniPlayer( - usePreviewMiniPlayerStore.getState().byThreadKey, - threadRef, - ); - const presentation = useBrowserSurfaceStore.getState().byTabId[tabId]; throw new PreviewAutomationVisibilityTimeoutError({ requestId, environmentId: threadRef.environmentId, threadId: threadRef.threadId, tabId, timeoutMs, - activeSurfaceId: miniPlayer ? `mini-player:${miniPlayer.tabId}` : panel.activeSurfaceId, - rightPanelOpen: panel.isOpen, - surfaceRegistered: presentation !== undefined, - presentationRectAvailable: presentation?.rect !== null && presentation?.rect !== undefined, + ...readPreviewAutomationPresentationDiagnostics(threadRef, tabId), }); }; diff --git a/apps/web/src/components/preview/previewAutomationErrors.ts b/apps/web/src/components/preview/previewAutomationErrors.ts index 68fc90203f1..c0c1a954169 100644 --- a/apps/web/src/components/preview/previewAutomationErrors.ts +++ b/apps/web/src/components/preview/previewAutomationErrors.ts @@ -83,8 +83,12 @@ export class PreviewAutomationVisibilityTimeoutError extends Schema.TaggedErrorC threadId: ThreadId, tabId: PreviewTabId, timeoutMs: Schema.Int, + activeSurfaceKind: Schema.optional(Schema.Literals(["inline-preview", "right-panel", "none"])), activeSurfaceId: Schema.optional(Schema.NullOr(Schema.String)), + inlinePreviewOpen: Schema.optional(Schema.Boolean), + inlinePreviewTabId: Schema.optional(Schema.NullOr(PreviewTabId)), rightPanelOpen: Schema.optional(Schema.Boolean), + rightPanelSurfaceId: Schema.optional(Schema.NullOr(Schema.String)), surfaceRegistered: Schema.optional(Schema.Boolean), presentationRectAvailable: Schema.optional(Schema.Boolean), }, diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index f836a2cdb61..2f4683a5416 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -9,9 +9,11 @@ import { resetPreviewStateForTests, } from "~/previewStateStore"; import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; +import { useRightPanelStore } from "~/rightPanelStore"; import { isPreviewAutomationTabPresented, + readPreviewAutomationPresentationDiagnostics, revealPreviewAutomationTab, withPreviewAutomationBackgroundPresentation, waitForPreviewAutomationBackgroundPresentation, @@ -32,6 +34,7 @@ describe("preview automation presentation", () => { beforeEach(() => { resetPreviewStateForTests(); usePreviewMiniPlayerStore.setState({ byThreadKey: {} }); + useRightPanelStore.setState({ byThreadKey: {} }); useBrowserSurfaceStore.setState({ byTabId: {}, backgroundCaptureCountByTabId: {} }); }); @@ -48,17 +51,46 @@ describe("preview automation presentation", () => { tabId: "tab-1", }); expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(false); + expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1")).toEqual({ + activeSurfaceKind: "inline-preview", + activeSurfaceId: "tab-1", + inlinePreviewOpen: true, + inlinePreviewTabId: "tab-1", + rightPanelOpen: false, + rightPanelSurfaceId: null, + surfaceRegistered: false, + presentationRectAvailable: false, + }); const surface = acquireBrowserSurface("tab-1"); surface.present({ x: 0, y: 0, width: 800, height: 600 }, true); expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(true); + expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1")).toMatchObject({ + surfaceRegistered: true, + presentationRectAvailable: true, + }); usePreviewMiniPlayerStore.getState().open(threadRef, "tab-2"); expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(false); surface.release(); }); + it("reports a right-panel presentation separately from the inline preview", () => { + useRightPanelStore.getState().openBrowser(threadRef, "tab-panel"); + + expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested")).toEqual({ + activeSurfaceKind: "right-panel", + activeSurfaceId: "browser:tab-panel", + inlinePreviewOpen: false, + inlinePreviewTabId: null, + rightPanelOpen: true, + rightPanelSurfaceId: "browser:tab-panel", + surfaceRegistered: false, + presentationRectAvailable: false, + }); + }); + it("uses the operation budget when background staging does not render", async () => { vi.useFakeTimers(); vi.stubGlobal("document", { diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index 0a862efaa6d..ae8039b7567 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -63,6 +63,31 @@ export function revealPreviewAutomationTab(ref: ScopedThreadRef, tabId: string): usePreviewMiniPlayerStore.getState().open(ref, tabId); } +export function readPreviewAutomationPresentationDiagnostics(ref: ScopedThreadRef, tabId: string) { + const panel = selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, ref); + const miniPlayer = selectThreadPreviewMiniPlayer( + usePreviewMiniPlayerStore.getState().byThreadKey, + ref, + ); + const presentation = useBrowserSurfaceStore.getState().byTabId[tabId]; + const activeSurfaceKind = + miniPlayer !== null + ? ("inline-preview" as const) + : panel.isOpen && panel.activeSurfaceId !== null + ? ("right-panel" as const) + : ("none" as const); + return { + activeSurfaceKind, + activeSurfaceId: miniPlayer?.tabId ?? panel.activeSurfaceId, + inlinePreviewOpen: miniPlayer !== null, + inlinePreviewTabId: miniPlayer?.tabId ?? null, + rightPanelOpen: panel.isOpen, + rightPanelSurfaceId: panel.activeSurfaceId, + surfaceRegistered: presentation !== undefined, + presentationRectAvailable: presentation?.rect !== null && presentation?.rect !== undefined, + }; +} + export function isPreviewAutomationTabPresented(ref: ScopedThreadRef, tabId: string): boolean { const panel = selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, ref); const miniPlayer = selectThreadPreviewMiniPlayer( diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts index 7e52de78829..8fd3b682feb 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts @@ -315,8 +315,12 @@ describe("previewAutomationRequestConsumer", () => { threadId, tabId, timeoutMs: 2_000, - activeSurfaceId: "browser:tab-1", - rightPanelOpen: true, + activeSurfaceKind: "inline-preview", + activeSurfaceId: "tab-1", + inlinePreviewOpen: true, + inlinePreviewTabId: "tab-1", + rightPanelOpen: false, + rightPanelSurfaceId: null, surfaceRegistered: true, presentationRectAvailable: false, }); @@ -334,8 +338,12 @@ describe("previewAutomationRequestConsumer", () => { detail: { tabId: "tab-1", timeoutMs: 2_000, - activeSurfaceId: "browser:tab-1", - rightPanelOpen: true, + activeSurfaceKind: "inline-preview", + activeSurfaceId: "tab-1", + inlinePreviewOpen: true, + inlinePreviewTabId: "tab-1", + rightPanelOpen: false, + rightPanelSurfaceId: null, surfaceRegistered: true, presentationRectAvailable: false, }, From f5d3caed0ea9ee2c9621435a74959a9a37957e6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Mon, 27 Jul 2026 15:15:57 +0100 Subject: [PATCH 18/43] Refresh preview reliability branch documentation --- BRANCH_DETAILS.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 7861c689bc0..899face8aa5 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -7,13 +7,13 @@ Expected behavior: - Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session when initialization or an acquired permit may have left a CDP command pending. A request that times out while queued behind another action does not detach that action's shared debugger session. - Click, type, and wait operations propagate their caller-supplied timeout into the desktop control-session boundary. Operations without a caller timeout retain the bounded desktop default. - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. -- Desktop preview guests no longer create their CDP debugger session eagerly when a webview registers. Session initialization is lazy and included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. Closing detached DevTools restores an explicit color-scheme override through the separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. -- Inactive preview webviews remain mounted, retain their declared viewport, and stay CSS-visible while positioned outside the human-visible panel. This preserves their runtime and semantic or input automation without selecting them. Background snapshot capture uses a reference-counted presentation lease and always restores the offscreen position afterward; navigation, color-scheme changes, evaluation, waits, recording, and input operations remain compatible with the selected inline preview or right-panel surface without acquiring that lease. Snapshot staging does not change the human-selected preview surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target in either the inline mini-player or right panel while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: capture staging falls back to a deterministic rectangle fitted inside the renderer viewport. +- Desktop preview guests create their CDP debugger session lazily, with initialization included in the automation operation deadline. `apps/desktop/src/preview/Manager.ts` restores a persisted non-system color-scheme override after webview registration or detached DevTools closes through a separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. +- Inactive preview webviews remain mounted, retain their declared viewport, and stay CSS-visible while positioned outside the human-visible panel. This preserves their runtime and semantic or input automation without selecting them. `apps/web/src/browser/browserSurfaceStore.ts` reference-counts background snapshot presentation independently from the normal surface lease, while `HostedBrowserWebview.tsx` composes that state with fitted-source content and corner-radius presentation. Only snapshot staging acquires the background lease; navigation, color-scheme changes, evaluation, waits, and input operations remain offscreen. Staging always restores the offscreen position and does not change the human-selected inline preview or right-panel surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: capture staging falls back to a deterministic rectangle fitted inside the renderer viewport. - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. -- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface, which presentation kind is active, whether the requested browser surface was registered, and whether it had a presentation rectangle. -- A newly created preview tab applies its server snapshot and assigned tab id, initiates any requested selection, and acknowledges server-side creation immediately without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for upstream's `open` input. -- The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes follow upstream's single-origin architecture: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode continues to pin `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. +- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. `previewAutomationPresentation.ts` reads both presentation stores, and `previewAutomationErrors.ts` reports the inline preview's selected tab, the right panel's active surface, the active presentation kind, surface registration, and rectangle availability. +- A newly created preview tab applies its server snapshot and assigned tab id, initiates any requested selection, and acknowledges server-side creation immediately without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. For reused tabs, `PreviewAutomationHosts.tsx` and `previewAutomationOpenReadiness.ts` select the matching inline mini-player, retain overlay and navigation readiness checks, and require stable presentation when visibility is requested. While a request remains pending, it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. The deprecated `show` input remains an alias for `open`. +- The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. Current limitations: From 538aebaf454c1f593ef440f74d7187b50b6e8e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Mon, 27 Jul 2026 17:20:40 +0100 Subject: [PATCH 19/43] Stabilize preview presentation diagnostics - Preserve namespaced active surface identifiers across inline previews - Keep timeout diagnostics consistent for hidden and competing surfaces - Restore complete behavior-level branch documentation --- BRANCH_DETAILS.md | 8 +-- .../previewAutomationPresentation.test.ts | 53 ++++++++++++++++++- .../preview/previewAutomationPresentation.ts | 44 +++++++++++---- .../previewAutomationRequestConsumer.test.ts | 4 +- 4 files changed, 91 insertions(+), 18 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 899face8aa5..02fd5846d90 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -7,12 +7,12 @@ Expected behavior: - Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session when initialization or an acquired permit may have left a CDP command pending. A request that times out while queued behind another action does not detach that action's shared debugger session. - Click, type, and wait operations propagate their caller-supplied timeout into the desktop control-session boundary. Operations without a caller timeout retain the bounded desktop default. - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. -- Desktop preview guests create their CDP debugger session lazily, with initialization included in the automation operation deadline. `apps/desktop/src/preview/Manager.ts` restores a persisted non-system color-scheme override after webview registration or detached DevTools closes through a separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. -- Inactive preview webviews remain mounted, retain their declared viewport, and stay CSS-visible while positioned outside the human-visible panel. This preserves their runtime and semantic or input automation without selecting them. `apps/web/src/browser/browserSurfaceStore.ts` reference-counts background snapshot presentation independently from the normal surface lease, while `HostedBrowserWebview.tsx` composes that state with fitted-source content and corner-radius presentation. Only snapshot staging acquires the background lease; navigation, color-scheme changes, evaluation, waits, and input operations remain offscreen. Staging always restores the offscreen position and does not change the human-selected inline preview or right-panel surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: capture staging falls back to a deterministic rectangle fitted inside the renderer viewport. +- Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. A persisted non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. +- Inactive preview webviews remain mounted, retain their declared viewport, and stay CSS-visible while positioned outside the human-visible panel. This preserves their runtime and semantic or input automation without selecting them. Background snapshot presentation is reference-counted independently from the normal surface lease and composes with fitted-source content and corner-radius presentation. Only snapshot staging acquires the background lease; navigation, color-scheme changes, evaluation, waits, recording, and input operations remain compatible with the selected inline preview or right-panel surface without acquiring that lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: capture staging falls back to a deterministic rectangle fitted inside the renderer viewport. - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. -- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. `previewAutomationPresentation.ts` reads both presentation stores, and `previewAutomationErrors.ts` reports the inline preview's selected tab, the right panel's active surface, the active presentation kind, surface registration, and rectangle availability. -- A newly created preview tab applies its server snapshot and assigned tab id, initiates any requested selection, and acknowledges server-side creation immediately without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. For reused tabs, `PreviewAutomationHosts.tsx` and `previewAutomationOpenReadiness.ts` select the matching inline mini-player, retain overlay and navigation readiness checks, and require stable presentation when visibility is requested. While a request remains pending, it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. The deprecated `show` input remains an alias for `open`. +- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. +- A newly created preview tab applies its server snapshot and assigned tab id, initiates any requested selection, and acknowledges server-side creation immediately without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for `open`. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index 2f4683a5416..ec9bbec2ab9 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -53,7 +53,7 @@ describe("preview automation presentation", () => { expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(false); expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1")).toEqual({ activeSurfaceKind: "inline-preview", - activeSurfaceId: "tab-1", + activeSurfaceId: "mini-player:tab-1", inlinePreviewOpen: true, inlinePreviewTabId: "tab-1", rightPanelOpen: false, @@ -73,9 +73,60 @@ describe("preview automation presentation", () => { usePreviewMiniPlayerStore.getState().open(threadRef, "tab-2"); expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(false); + expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1")).toEqual({ + activeSurfaceKind: "inline-preview", + activeSurfaceId: "mini-player:tab-2", + inlinePreviewOpen: true, + inlinePreviewTabId: "tab-2", + rightPanelOpen: false, + rightPanelSurfaceId: null, + surfaceRegistered: true, + presentationRectAvailable: true, + }); surface.release(); }); + it("reports presentation precedence and hidden retained panel state", () => { + expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested")).toEqual({ + activeSurfaceKind: "none", + activeSurfaceId: null, + inlinePreviewOpen: false, + inlinePreviewTabId: null, + rightPanelOpen: false, + rightPanelSurfaceId: null, + surfaceRegistered: false, + presentationRectAvailable: false, + }); + + useRightPanelStore.getState().openBrowser(threadRef, "tab-panel"); + usePreviewMiniPlayerStore.getState().open(threadRef, "tab-inline"); + + expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested")).toEqual({ + activeSurfaceKind: "inline-preview", + activeSurfaceId: "mini-player:tab-inline", + inlinePreviewOpen: true, + inlinePreviewTabId: "tab-inline", + rightPanelOpen: true, + rightPanelSurfaceId: "browser:tab-panel", + surfaceRegistered: false, + presentationRectAvailable: false, + }); + + usePreviewMiniPlayerStore.getState().close(threadRef); + useRightPanelStore.getState().close(threadRef); + + expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested")).toEqual({ + activeSurfaceKind: "none", + activeSurfaceId: null, + inlinePreviewOpen: false, + inlinePreviewTabId: null, + rightPanelOpen: false, + rightPanelSurfaceId: "browser:tab-panel", + surfaceRegistered: false, + presentationRectAvailable: false, + }); + }); + it("reports a right-panel presentation separately from the inline preview", () => { useRightPanelStore.getState().openBrowser(threadRef, "tab-panel"); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index ae8039b7567..0d21396691b 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -9,6 +9,7 @@ import { setActivePreviewTab } from "~/previewStateStore"; import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelStore"; import { PreviewAutomationBackgroundPresentationTimeoutError } from "./previewAutomationErrors"; +import type { PreviewAutomationVisibilityTimeoutError } from "./previewAutomationErrors"; interface PreviewAutomationBackgroundPresentationInput { readonly threadRef: ScopedThreadRef; @@ -17,6 +18,20 @@ interface PreviewAutomationBackgroundPresentationInput { readonly timeoutMs: number; } +type PreviewAutomationPresentationDiagnostics = Required< + Pick< + PreviewAutomationVisibilityTimeoutError, + | "activeSurfaceKind" + | "activeSurfaceId" + | "inlinePreviewOpen" + | "inlinePreviewTabId" + | "rightPanelOpen" + | "rightPanelSurfaceId" + | "surfaceRegistered" + | "presentationRectAvailable" + > +>; + const PREVIEW_AUTOMATION_COMPOSITOR_FRAME_FALLBACK_MS = 16; function backgroundPresentationTimeoutError( @@ -63,13 +78,21 @@ export function revealPreviewAutomationTab(ref: ScopedThreadRef, tabId: string): usePreviewMiniPlayerStore.getState().open(ref, tabId); } -export function readPreviewAutomationPresentationDiagnostics(ref: ScopedThreadRef, tabId: string) { +function readPreviewAutomationPresentation(ref: ScopedThreadRef, tabId: string) { const panel = selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, ref); const miniPlayer = selectThreadPreviewMiniPlayer( usePreviewMiniPlayerStore.getState().byThreadKey, ref, ); const presentation = useBrowserSurfaceStore.getState().byTabId[tabId]; + return { panel, miniPlayer, presentation }; +} + +export function readPreviewAutomationPresentationDiagnostics( + ref: ScopedThreadRef, + tabId: string, +): PreviewAutomationPresentationDiagnostics { + const { panel, miniPlayer, presentation } = readPreviewAutomationPresentation(ref, tabId); const activeSurfaceKind = miniPlayer !== null ? ("inline-preview" as const) @@ -78,27 +101,26 @@ export function readPreviewAutomationPresentationDiagnostics(ref: ScopedThreadRe : ("none" as const); return { activeSurfaceKind, - activeSurfaceId: miniPlayer?.tabId ?? panel.activeSurfaceId, + activeSurfaceId: + miniPlayer !== null + ? `mini-player:${miniPlayer.tabId}` + : panel.isOpen + ? panel.activeSurfaceId + : null, inlinePreviewOpen: miniPlayer !== null, inlinePreviewTabId: miniPlayer?.tabId ?? null, rightPanelOpen: panel.isOpen, rightPanelSurfaceId: panel.activeSurfaceId, surfaceRegistered: presentation !== undefined, - presentationRectAvailable: presentation?.rect !== null && presentation?.rect !== undefined, + presentationRectAvailable: presentation?.rect != null, }; } export function isPreviewAutomationTabPresented(ref: ScopedThreadRef, tabId: string): boolean { - const panel = selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, ref); - const miniPlayer = selectThreadPreviewMiniPlayer( - usePreviewMiniPlayerStore.getState().byThreadKey, - ref, - ); + const { panel, miniPlayer, presentation } = readPreviewAutomationPresentation(ref, tabId); const requestedSurfaceIsActive = (panel.isOpen && panel.activeSurfaceId === `browser:${tabId}`) || miniPlayer?.tabId === tabId; - return ( - requestedSurfaceIsActive && (useBrowserSurfaceStore.getState().byTabId[tabId]?.visible ?? false) - ); + return requestedSurfaceIsActive && (presentation?.visible ?? false); } export async function waitForPreviewAutomationBackgroundPresentation( diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts index 8fd3b682feb..23451d4a622 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts @@ -316,7 +316,7 @@ describe("previewAutomationRequestConsumer", () => { tabId, timeoutMs: 2_000, activeSurfaceKind: "inline-preview", - activeSurfaceId: "tab-1", + activeSurfaceId: "mini-player:tab-1", inlinePreviewOpen: true, inlinePreviewTabId: "tab-1", rightPanelOpen: false, @@ -339,7 +339,7 @@ describe("previewAutomationRequestConsumer", () => { tabId: "tab-1", timeoutMs: 2_000, activeSurfaceKind: "inline-preview", - activeSurfaceId: "tab-1", + activeSurfaceId: "mini-player:tab-1", inlinePreviewOpen: true, inlinePreviewTabId: "tab-1", rightPanelOpen: false, From 41db26057d927c866d572d4881cb0dea198be129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Mon, 27 Jul 2026 20:35:45 +0100 Subject: [PATCH 20/43] Document upstream preview capture ownership boundary --- BRANCH_DETAILS.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 02fd5846d90..6c614028825 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -1,6 +1,17 @@ # Preview Automation Reliability -Product-native preview automation is bounded and recoverable across the web host, MCP server, and Electron CDP controller. The branch-specific layer covers operation deadlines, control-session recovery, background capture presentation, and degraded semantic snapshots. +Product-native preview automation is bounded and recoverable across the web host, MCP server, and Electron CDP controller. The branch-specific layer covers operation deadlines, control-session recovery, one-shot automation snapshot presentation, and degraded semantic snapshots. + +## Upstream ownership boundary + +Upstream commit `f4c394323` (`Add background preview capture and picture-in-picture support`) supersedes the earlier assumption that this branch owns generic background preview capture. Upstream now owns: + +- keeping inactive preview guests mounted and CSS-visible outside the human-visible panel; +- the shared `webContents.capturePage()` frame loop used by recording and picture-in-picture; +- inline preview mini-player presentation, fitted-source layout, corner radii, and crash recovery; and +- background-only preview creation through `open: false`. + +This branch does not add a second recording/PiP capture lifecycle or another hidden-preview lifetime mechanism. Its remaining background-specific code is limited to bounded, one-shot automation snapshots. That distinction is necessary: a fresh offscreen guest did not produce a frame or settle upstream recording startup within eight seconds, while staging the same guest at effectively transparent opacity produced a complete screenshot and semantic snapshot in under 100 milliseconds. Upstream's retrying frame loop therefore does not replace the automation snapshot presentation lease, exact-target CDP capture, request deadline propagation, or nullable semantic fallback. Expected behavior: @@ -8,7 +19,7 @@ Expected behavior: - Click, type, and wait operations propagate their caller-supplied timeout into the desktop control-session boundary. Operations without a caller timeout retain the bounded desktop default. - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. A persisted non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. -- Inactive preview webviews remain mounted, retain their declared viewport, and stay CSS-visible while positioned outside the human-visible panel. This preserves their runtime and semantic or input automation without selecting them. Background snapshot presentation is reference-counted independently from the normal surface lease and composes with fitted-source content and corner-radius presentation. Only snapshot staging acquires the background lease; navigation, color-scheme changes, evaluation, waits, recording, and input operations remain compatible with the selected inline preview or right-panel surface without acquiring that lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: capture staging falls back to a deterministic rectangle fitted inside the renderer viewport. +- Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. From 59795c11c45e3a8585586ef25c09dc39deaa9f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Tue, 28 Jul 2026 09:45:43 +0100 Subject: [PATCH 21/43] Isolate desktop user data in dev runner --- BRANCH_DETAILS.md | 17 ++++++++++++- .../src/app/DesktopAppIdentity.test.ts | 25 +++++++++++++++++++ apps/desktop/src/app/DesktopAppIdentity.ts | 4 +++ apps/desktop/src/app/DesktopConfig.ts | 1 + .../src/app/DesktopEnvironment.test.ts | 2 ++ apps/desktop/src/app/DesktopEnvironment.ts | 2 ++ scripts/dev-runner.test.ts | 4 +++ scripts/dev-runner.ts | 7 ++++++ 8 files changed, 61 insertions(+), 1 deletion(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 6c614028825..f26aa43dfe3 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -30,9 +30,18 @@ Expected behavior: Current limitations: - Electron screenshot capture can still be unavailable when both the bounded CDP compositor capture and hidden `capturePage` fallback fail. The intended degraded result remains a usable semantic snapshot with `screenshot: null`, not raster evidence. +- The current dev-host pass exercised pairing plus direct CDP semantic and raster capture without the Electron host interface disappearing, but it did not re-run the product-native hidden, non-selected `preview_snapshot` path in that same host. The spawned agents' preview tools remained attached to the installed T3 Code environment rather than the worktree dev desktop, so that exact end-to-end claim still requires correctly routed preview automation. + +Current uncommitted follow-up: + +- `dev:desktop` now derives `T3CODE_DESKTOP_USER_DATA_DIR=/userdata/electron` whenever the runner has an explicit base directory. Desktop configuration resolves that override to an absolute path, and app identity uses it before legacy migration or the normal Electron user-data default. This keeps an isolated worktree dev desktop from reusing an installed or earlier development profile whose incompatible IndexedDB schema can prevent the renderer from starting. Packaged/default startup remains unchanged when no override is supplied. +- These seven source and test-file changes are intentionally uncommitted while runtime verification continues. Primary files: +- `apps/desktop/src/app/DesktopAppIdentity.ts` +- `apps/desktop/src/app/DesktopConfig.ts` +- `apps/desktop/src/app/DesktopEnvironment.ts` - `apps/desktop/src/preview/Manager.ts` - `apps/desktop/src/ipc/methods/preview.ts` - `apps/desktop/src/preload.ts` @@ -53,12 +62,18 @@ Primary files: - `packages/contracts/src/ipc.ts` - `scripts/dev-runner.ts` -Focused regression coverage lives in `scripts/dev-runner.test.ts`, `apps/desktop/src/preview/Manager.test.ts`, `apps/server/src/mcp/McpHttpServer.test.ts`, `apps/web/src/browser/browserSurfaceStore.test.ts`, `apps/web/src/browser/hostedBrowserWebviewStyle.test.ts`, `apps/web/src/components/auth/PairingRouteSurface.logic.test.ts`, `apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationPresentation.test.ts`, `apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts`, `packages/contracts/src/ipc.test.ts`, and `packages/contracts/src/preview.test.ts`. +Focused regression coverage lives in `scripts/dev-runner.test.ts`, `apps/desktop/src/app/DesktopAppIdentity.test.ts`, `apps/desktop/src/app/DesktopEnvironment.test.ts`, `apps/desktop/src/preview/Manager.test.ts`, `apps/server/src/mcp/McpHttpServer.test.ts`, `apps/web/src/browser/browserSurfaceStore.test.ts`, `apps/web/src/browser/hostedBrowserWebviewStyle.test.ts`, `apps/web/src/components/auth/PairingRouteSurface.logic.test.ts`, `apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationPresentation.test.ts`, `apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts`, `packages/contracts/src/ipc.test.ts`, and `packages/contracts/src/preview.test.ts`. ```sh vp test run scripts/dev-runner.test.ts apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/web/src/browser/browserSurfaceStore.test.ts apps/web/src/browser/hostedBrowserWebviewStyle.test.ts apps/web/src/components/auth/PairingRouteSurface.logic.test.ts apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts apps/web/src/components/preview/previewAutomationPresentation.test.ts apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts packages/contracts/src/ipc.test.ts packages/contracts/src/preview.test.ts ``` +Verification completed for the current uncommitted follow-up: + +- `vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.test.ts apps/desktop/src/app/DesktopEnvironment.test.ts` passed all 77 tests. +- `vp run --filter @t3tools/desktop typecheck` and `vp run --filter ./scripts typecheck` completed without type errors. The desktop check still printed two Effect suggestions in unchanged files. +- A worktree `dev:desktop` instance using the isolated user-data override paired successfully and stayed usable through repeated renderer/CDP inspection and raster capture. An archive worktree client loaded its seeded project against that host, and the prior IndexedDB `VersionError` and host-interface disappearance did not recur in those exercised flows. + ## Development Ports Preferred ports when explicitly selecting offset 11: diff --git a/apps/desktop/src/app/DesktopAppIdentity.test.ts b/apps/desktop/src/app/DesktopAppIdentity.test.ts index 3c95b266bc1..f9b1e7a0519 100644 --- a/apps/desktop/src/app/DesktopAppIdentity.test.ts +++ b/apps/desktop/src/app/DesktopAppIdentity.test.ts @@ -141,6 +141,31 @@ const withIdentity = ( }; describe("DesktopAppIdentity", () => { + it.effect("uses an explicit desktop user-data directory without probing legacy state", () => + withIdentity( + Effect.gen(function* () { + const identity = yield* DesktopAppIdentity.DesktopAppIdentity; + const userDataPath = yield* identity.resolveUserDataPath; + + assert.equal(userDataPath, "/tmp/t3-desktop-profile"); + }), + { + environment: { + env: { + T3CODE_DESKTOP_USER_DATA_DIR: " /tmp/t3-desktop-profile ", + }, + }, + legacyPathProbeError: PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "exists", + description: "legacy path must not be probed", + pathOrDescriptor: "/Users/alice/Library/Application Support/T3 Code (Alpha)", + }), + }, + ), + ); + it.effect("keeps using the legacy userData path when it already exists", () => withIdentity( Effect.gen(function* () { diff --git a/apps/desktop/src/app/DesktopAppIdentity.ts b/apps/desktop/src/app/DesktopAppIdentity.ts index 385e694338d..3477a00dea1 100644 --- a/apps/desktop/src/app/DesktopAppIdentity.ts +++ b/apps/desktop/src/app/DesktopAppIdentity.ts @@ -91,6 +91,10 @@ export const make = Effect.gen(function* () { }); const resolveUserDataPath = Effect.gen(function* () { + if (Option.isSome(environment.desktopUserDataDir)) { + return environment.desktopUserDataDir.value; + } + const legacyPath = environment.path.join( environment.appDataDirectory, environment.legacyUserDataDirName, diff --git a/apps/desktop/src/app/DesktopConfig.ts b/apps/desktop/src/app/DesktopConfig.ts index 4bf6b513306..99098a28289 100644 --- a/apps/desktop/src/app/DesktopConfig.ts +++ b/apps/desktop/src/app/DesktopConfig.ts @@ -37,6 +37,7 @@ export const DesktopConfig = Config.all({ xdgConfigHome: trimmedString("XDG_CONFIG_HOME"), t3Home: trimmedString("T3CODE_HOME"), devServerUrl: Config.url("VITE_DEV_SERVER_URL").pipe(Config.option), + desktopUserDataDir: trimmedString("T3CODE_DESKTOP_USER_DATA_DIR"), appUserModelIdOverride: trimmedString("T3CODE_DESKTOP_APP_USER_MODEL_ID"), devRemoteT3ServerEntryPath: trimmedString("T3CODE_DEV_REMOTE_T3_SERVER_ENTRY_PATH"), configuredBackendPort: Config.port("T3CODE_PORT").pipe(Config.option), diff --git a/apps/desktop/src/app/DesktopEnvironment.test.ts b/apps/desktop/src/app/DesktopEnvironment.test.ts index 15d23f8e152..2c95ceb85cb 100644 --- a/apps/desktop/src/app/DesktopEnvironment.test.ts +++ b/apps/desktop/src/app/DesktopEnvironment.test.ts @@ -41,6 +41,7 @@ describe("DesktopEnvironment", () => { {}, { T3CODE_HOME: " /tmp/t3 ", + T3CODE_DESKTOP_USER_DATA_DIR: " /tmp/t3/userdata/electron ", T3CODE_COMMIT_HASH: " 0123456789abcdef ", T3CODE_PORT: "4949", VITE_DEV_SERVER_URL: "http://localhost:5173", @@ -73,6 +74,7 @@ describe("DesktopEnvironment", () => { Option.map(environment.devServerUrl, (url) => url.href), Option.some("http://localhost:5173/"), ); + assert.deepEqual(environment.desktopUserDataDir, Option.some("/tmp/t3/userdata/electron")); assert.deepEqual(environment.devRemoteT3ServerEntryPath, Option.some("/remote/server.mjs")); assert.deepEqual(environment.configuredBackendPort, Option.some(4949)); assert.deepEqual(environment.commitHashOverride, Option.some("0123456789abcdef")); diff --git a/apps/desktop/src/app/DesktopEnvironment.ts b/apps/desktop/src/app/DesktopEnvironment.ts index c991f5b39d6..9a0dc8c498b 100644 --- a/apps/desktop/src/app/DesktopEnvironment.ts +++ b/apps/desktop/src/app/DesktopEnvironment.ts @@ -56,6 +56,7 @@ export class DesktopEnvironment extends Context.Service< readonly preloadPath: string; readonly appUpdateYmlPath: string; readonly devServerUrl: Option.Option; + readonly desktopUserDataDir: Option.Option; readonly devRemoteT3ServerEntryPath: Option.Option; readonly configuredBackendPort: Option.Option; readonly commitHashOverride: Option.Option; @@ -193,6 +194,7 @@ const make = Effect.fn("desktop.environment.make")(function* ( ? path.join(resourcesPath, "app-update.yml") : path.join(input.appPath, "dev-app-update.yml"), devServerUrl, + desktopUserDataDir: Option.map(config.desktopUserDataDir, path.resolve), devRemoteT3ServerEntryPath: config.devRemoteT3ServerEntryPath, configuredBackendPort: config.configuredBackendPort, commitHashOverride: config.commitHashOverride, diff --git a/scripts/dev-runner.test.ts b/scripts/dev-runner.test.ts index bb6a62ab3a2..fc2db9131c5 100644 --- a/scripts/dev-runner.test.ts +++ b/scripts/dev-runner.test.ts @@ -318,6 +318,10 @@ it.layer(NodeServices.layer)("dev-runner", (it) => { }); assert.equal(env.T3CODE_HOME, path.resolve("/tmp/my-t3")); + assert.equal( + env.T3CODE_DESKTOP_USER_DATA_DIR, + path.resolve("/tmp/my-t3/userdata/electron"), + ); assert.equal(env.PORT, "5733"); assert.equal(env.VITE_DEV_SERVER_URL, "http://127.0.0.1:5733"); assert.equal(env.HOST, "127.0.0.1"); diff --git a/scripts/dev-runner.ts b/scripts/dev-runner.ts index b7e23fad7e7..29be4a12eca 100644 --- a/scripts/dev-runner.ts +++ b/scripts/dev-runner.ts @@ -318,6 +318,7 @@ export function createDevRunnerEnv({ devUrl, }: CreateDevRunnerEnvInput): Effect.Effect { return Effect.gen(function* () { + const path = yield* Path.Path; const serverPort = port ?? BASE_SERVER_PORT + serverOffset; const webPort = BASE_WEB_PORT + webOffset; // Precedence (--home-dir > worktree .t3 > ambient T3CODE_HOME) is resolved @@ -338,6 +339,12 @@ export function createDevRunnerEnv({ delete output.T3CODE_HOME; } + if (isDesktopMode && configuredBaseDir !== undefined) { + output.T3CODE_DESKTOP_USER_DATA_DIR = path.join(resolvedBaseDir, "userdata", "electron"); + } else { + delete output.T3CODE_DESKTOP_USER_DATA_DIR; + } + if (!isDesktopMode) { output.T3CODE_PORT = String(serverPort); // HOST is Vite's own bind address, and the desktop branch below is the From 0ccb20073e576d3c7a262dc44d2244f916d32226 Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Tue, 28 Jul 2026 15:01:39 +0100 Subject: [PATCH 22/43] Update preview presentation tests for runtime tab ids --- apps/web/src/browser/browserSurfaceStore.test.ts | 1 + .../components/preview/previewAutomationPresentation.test.ts | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/web/src/browser/browserSurfaceStore.test.ts b/apps/web/src/browser/browserSurfaceStore.test.ts index 07e6ee8847c..33d7da7bd41 100644 --- a/apps/web/src/browser/browserSurfaceStore.test.ts +++ b/apps/web/src/browser/browserSurfaceStore.test.ts @@ -45,6 +45,7 @@ describe("browserSurfaceStore", () => { expect(first).toEqual({ byTabId: state.byTabId, backgroundCapture: true, + content: null, cornerRadius: 0, fitSourceContent: false, fittedSourceContent: null, diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index bb907b4ad3e..aec7c42a9f0 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -3,6 +3,7 @@ import { EnvironmentId, type PreviewSessionSnapshot, ThreadId } from "@t3tools/c import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; import { acquireBrowserSurface, useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; +import { previewRuntimeTabId } from "~/browser/previewRuntimeTabId"; import { applyPreviewServerSnapshot, readThreadPreviewState, @@ -62,7 +63,9 @@ describe("preview automation presentation", () => { presentationRectAvailable: false, }); - const surface = acquireBrowserSurface("tab-1"); + const surface = acquireBrowserSurface( + previewRuntimeTabId(threadRef, readThreadPreviewState(threadRef).serverEpoch, "tab-1"), + ); surface.present({ x: 0, y: 0, width: 800, height: 600 }, true); expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(true); From 5777b0f5ff2a3b902f3ca70e8d6580513228705b Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Tue, 28 Jul 2026 15:37:27 +0100 Subject: [PATCH 23/43] Require runtime ids for preview presentation --- .../preview/PreviewAutomationHosts.tsx | 6 +- .../previewAutomationPresentation.test.ts | 56 ++++++++++++++----- .../preview/previewAutomationPresentation.ts | 34 ++++------- 3 files changed, 59 insertions(+), 37 deletions(-) diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 7e2c7288dd1..a1167083d99 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -126,6 +126,7 @@ const waitForBrowserSurfaceVisibility = async ( threadRef: ScopedThreadRef, requestId: string, tabId: string, + runtimeTabId: string, timeoutMs: number, ): Promise => { const deadline = Date.now() + timeoutMs; @@ -133,7 +134,7 @@ const waitForBrowserSurfaceVisibility = async ( let presentedSince: number | null = null; while (Date.now() <= deadline) { const now = Date.now(); - if (isPreviewAutomationTabPresented(threadRef, tabId)) { + if (isPreviewAutomationTabPresented(threadRef, tabId, runtimeTabId)) { presentedSince ??= now; // Require the selection to survive multiple presentation updates. A // single transient `visible` frame can otherwise make open acknowledge @@ -153,7 +154,7 @@ const waitForBrowserSurfaceVisibility = async ( threadId: threadRef.threadId, tabId, timeoutMs, - ...readPreviewAutomationPresentationDiagnostics(threadRef, tabId), + ...readPreviewAutomationPresentationDiagnostics(threadRef, tabId, runtimeTabId), }); }; interface ExecutablePreviewWebview extends Element { @@ -532,6 +533,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) threadRef, request.requestId, activeTabId, + activeRuntimeTabId, remainingOperationBudget(), ); } diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index aec7c42a9f0..2e8c63d7712 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -42,6 +42,11 @@ describe("preview automation presentation", () => { it("selects the requested preview tab and its inline preview surface together", () => { applyPreviewServerSnapshot(threadRef, snapshot("tab-1", "2026-07-25T00:00:00.000Z")); applyPreviewServerSnapshot(threadRef, snapshot("tab-2", "2026-07-25T00:00:01.000Z")); + const tabOneRuntimeId = previewRuntimeTabId( + threadRef, + readThreadPreviewState(threadRef).serverEpoch, + "tab-1", + ); revealPreviewAutomationTab(threadRef, "tab-1"); @@ -51,8 +56,10 @@ describe("preview automation presentation", () => { ).toMatchObject({ tabId: "tab-1", }); - expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(false); - expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1")).toEqual({ + expect(isPreviewAutomationTabPresented(threadRef, "tab-1", tabOneRuntimeId)).toBe(false); + expect( + readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1", tabOneRuntimeId), + ).toEqual({ activeSurfaceKind: "inline-preview", activeSurfaceId: "mini-player:tab-1", inlinePreviewOpen: true, @@ -63,20 +70,33 @@ describe("preview automation presentation", () => { presentationRectAvailable: false, }); - const surface = acquireBrowserSurface( - previewRuntimeTabId(threadRef, readThreadPreviewState(threadRef).serverEpoch, "tab-1"), - ); + const serverIdSurface = acquireBrowserSurface("tab-1"); + serverIdSurface.present({ x: 0, y: 0, width: 800, height: 600 }, true); + expect(isPreviewAutomationTabPresented(threadRef, "tab-1", tabOneRuntimeId)).toBe(false); + expect( + readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1", tabOneRuntimeId), + ).toMatchObject({ + surfaceRegistered: false, + presentationRectAvailable: false, + }); + serverIdSurface.release(); + + const surface = acquireBrowserSurface(tabOneRuntimeId); surface.present({ x: 0, y: 0, width: 800, height: 600 }, true); - expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(true); - expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1")).toMatchObject({ + expect(isPreviewAutomationTabPresented(threadRef, "tab-1", tabOneRuntimeId)).toBe(true); + expect( + readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1", tabOneRuntimeId), + ).toMatchObject({ surfaceRegistered: true, presentationRectAvailable: true, }); usePreviewMiniPlayerStore.getState().open(threadRef, "tab-2"); - expect(isPreviewAutomationTabPresented(threadRef, "tab-1")).toBe(false); - expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1")).toEqual({ + expect(isPreviewAutomationTabPresented(threadRef, "tab-1", tabOneRuntimeId)).toBe(false); + expect( + readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1", tabOneRuntimeId), + ).toEqual({ activeSurfaceKind: "inline-preview", activeSurfaceId: "mini-player:tab-2", inlinePreviewOpen: true, @@ -90,7 +110,9 @@ describe("preview automation presentation", () => { }); it("reports presentation precedence and hidden retained panel state", () => { - expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested")).toEqual({ + expect( + readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested", "runtime-requested"), + ).toEqual({ activeSurfaceKind: "none", activeSurfaceId: null, inlinePreviewOpen: false, @@ -104,7 +126,9 @@ describe("preview automation presentation", () => { useRightPanelStore.getState().openBrowser(threadRef, "tab-panel"); usePreviewMiniPlayerStore.getState().open(threadRef, "tab-inline"); - expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested")).toEqual({ + expect( + readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested", "runtime-requested"), + ).toEqual({ activeSurfaceKind: "inline-preview", activeSurfaceId: "mini-player:tab-inline", inlinePreviewOpen: true, @@ -118,7 +142,9 @@ describe("preview automation presentation", () => { usePreviewMiniPlayerStore.getState().close(threadRef); useRightPanelStore.getState().close(threadRef); - expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested")).toEqual({ + expect( + readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested", "runtime-requested"), + ).toEqual({ activeSurfaceKind: "none", activeSurfaceId: null, inlinePreviewOpen: false, @@ -133,7 +159,9 @@ describe("preview automation presentation", () => { it("reports a right-panel presentation separately from the inline preview", () => { useRightPanelStore.getState().openBrowser(threadRef, "tab-panel"); - expect(readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested")).toEqual({ + expect( + readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested", "runtime-requested"), + ).toEqual({ activeSurfaceKind: "right-panel", activeSurfaceId: "browser:tab-panel", inlinePreviewOpen: false, @@ -158,6 +186,7 @@ describe("preview automation presentation", () => { threadRef, requestId: "request-background", tabId: "tab-background", + runtimeTabId: "runtime-background", timeoutMs: 40, }); const rejection = expect(presentation).rejects.toMatchObject({ @@ -190,6 +219,7 @@ describe("preview automation presentation", () => { threadRef, requestId: "request-foregrounded", tabId: "tab-foregrounded", + runtimeTabId: "tab-foregrounded", timeoutMs: 40, }); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index c46f7b02a53..2708767813d 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -4,9 +4,8 @@ import { acquireBrowserSurfaceBackgroundCapture, useBrowserSurfaceStore, } from "~/browser/browserSurfaceStore"; -import { previewRuntimeTabId } from "~/browser/previewRuntimeTabId"; import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; -import { readThreadPreviewState, setActivePreviewTab } from "~/previewStateStore"; +import { setActivePreviewTab } from "~/previewStateStore"; import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelStore"; import { PreviewAutomationBackgroundPresentationTimeoutError } from "./previewAutomationErrors"; @@ -16,7 +15,7 @@ interface PreviewAutomationBackgroundPresentationInput { readonly threadRef: ScopedThreadRef; readonly requestId: string; readonly tabId: string; - readonly runtimeTabId?: string; + readonly runtimeTabId: string; readonly timeoutMs: number; } @@ -80,38 +79,30 @@ export function revealPreviewAutomationTab(ref: ScopedThreadRef, tabId: string): usePreviewMiniPlayerStore.getState().open(ref, tabId); } -function resolvePresentationRuntimeTabId( - ref: ScopedThreadRef, - tabId: string, - runtimeTabId?: string, -): string { - if (runtimeTabId) return runtimeTabId; - const state = readThreadPreviewState(ref); - return state.sessions[tabId] ? previewRuntimeTabId(ref, state.serverEpoch, tabId) : tabId; -} - function readPreviewAutomationPresentation( ref: ScopedThreadRef, tabId: string, - runtimeTabId?: string, + runtimeTabId: string, ) { const panel = selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, ref); const miniPlayer = selectThreadPreviewMiniPlayer( usePreviewMiniPlayerStore.getState().byThreadKey, ref, ); - const presentation = - useBrowserSurfaceStore.getState().byTabId[ - resolvePresentationRuntimeTabId(ref, tabId, runtimeTabId) - ]; + const presentation = useBrowserSurfaceStore.getState().byTabId[runtimeTabId]; return { panel, miniPlayer, presentation }; } export function readPreviewAutomationPresentationDiagnostics( ref: ScopedThreadRef, tabId: string, + runtimeTabId: string, ): PreviewAutomationPresentationDiagnostics { - const { panel, miniPlayer, presentation } = readPreviewAutomationPresentation(ref, tabId); + const { panel, miniPlayer, presentation } = readPreviewAutomationPresentation( + ref, + tabId, + runtimeTabId, + ); const activeSurfaceKind = miniPlayer !== null ? ("inline-preview" as const) @@ -138,7 +129,7 @@ export function readPreviewAutomationPresentationDiagnostics( export function isPreviewAutomationTabPresented( ref: ScopedThreadRef, tabId: string, - runtimeTabId?: string, + runtimeTabId: string, ): boolean { const { panel, miniPlayer, presentation } = readPreviewAutomationPresentation( ref, @@ -164,8 +155,7 @@ export async function waitForPreviewAutomationBackgroundPresentation( document.querySelectorAll("[data-preview-viewport]"), ).find( (candidate) => - candidate.dataset["previewViewport"] === - resolvePresentationRuntimeTabId(input.threadRef, input.tabId, input.runtimeTabId) && + candidate.dataset["previewViewport"] === input.runtimeTabId && candidate.dataset["previewBackgroundCapture"] === "true", ); if (wrapper) { From 5bc0a7a6b72dd5386ee55321683452ba88d73f11 Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Tue, 28 Jul 2026 15:40:21 +0100 Subject: [PATCH 24/43] Refresh preview reliability branch details --- BRANCH_DETAILS.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index f26aa43dfe3..b70e5f8a36d 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -4,13 +4,15 @@ Product-native preview automation is bounded and recoverable across the web host ## Upstream ownership boundary -Upstream commit `f4c394323` (`Add background preview capture and picture-in-picture support`) supersedes the earlier assumption that this branch owns generic background preview capture. Upstream now owns: +Upstream commit `f4c394323` (`Add background preview capture and picture-in-picture support`) owns: - keeping inactive preview guests mounted and CSS-visible outside the human-visible panel; - the shared `webContents.capturePage()` frame loop used by recording and picture-in-picture; - inline preview mini-player presentation, fitted-source layout, corner radii, and crash recovery; and - background-only preview creation through `open: false`. +Upstream commit `32af2f002` (`fix(preview): stabilize PiP viewport identity`) owns epoch-scoped runtime guest identity and keeps PiP, recording, renderer surfaces, and Electron tabs aligned on that identity. + This branch does not add a second recording/PiP capture lifecycle or another hidden-preview lifetime mechanism. Its remaining background-specific code is limited to bounded, one-shot automation snapshots. That distinction is necessary: a fresh offscreen guest did not produce a frame or settle upstream recording startup within eight seconds, while staging the same guest at effectively transparent opacity produced a complete screenshot and semantic snapshot in under 100 milliseconds. Upstream's retrying frame loop therefore does not replace the automation snapshot presentation lease, exact-target CDP capture, request deadline propagation, or nullable semantic fallback. Expected behavior: @@ -19,23 +21,19 @@ Expected behavior: - Click, type, and wait operations propagate their caller-supplied timeout into the desktop control-session boundary. Operations without a caller timeout retain the bounded desktop default. - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. A persisted non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. -- Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. +- Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. -- A newly created preview tab applies its server snapshot and assigned tab id, initiates any requested selection, and acknowledges server-side creation immediately without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for `open`. +- A newly created preview tab applies its server snapshot and assigned tab id, applies the upstream 1280×800 automation viewport when the server snapshot still uses `fill`, initiates any requested selection, and acknowledges server-side creation without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for `open`. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. +- `dev:desktop` derives `T3CODE_DESKTOP_USER_DATA_DIR=/userdata/electron` whenever the runner has an explicit base directory. Desktop configuration resolves that override to an absolute path, and app identity uses it before legacy migration or the normal Electron user-data default. This keeps an isolated worktree dev desktop from reusing an installed or earlier development profile whose incompatible IndexedDB schema can prevent the renderer from starting. Packaged/default startup remains unchanged when no override is supplied. Current limitations: - Electron screenshot capture can still be unavailable when both the bounded CDP compositor capture and hidden `capturePage` fallback fail. The intended degraded result remains a usable semantic snapshot with `screenshot: null`, not raster evidence. -- The current dev-host pass exercised pairing plus direct CDP semantic and raster capture without the Electron host interface disappearing, but it did not re-run the product-native hidden, non-selected `preview_snapshot` path in that same host. The spawned agents' preview tools remained attached to the installed T3 Code environment rather than the worktree dev desktop, so that exact end-to-end claim still requires correctly routed preview automation. - -Current uncommitted follow-up: - -- `dev:desktop` now derives `T3CODE_DESKTOP_USER_DATA_DIR=/userdata/electron` whenever the runner has an explicit base directory. Desktop configuration resolves that override to an absolute path, and app identity uses it before legacy migration or the normal Electron user-data default. This keeps an isolated worktree dev desktop from reusing an installed or earlier development profile whose incompatible IndexedDB schema can prevent the renderer from starting. Packaged/default startup remains unchanged when no override is supplied. -- These seven source and test-file changes are intentionally uncommitted while runtime verification continues. +- End-to-end evidence does not currently cover the product-native hidden, non-selected `preview_snapshot` path in the isolated worktree dev desktop. The controlled web client is non-Electron and disables the Browser surface, while agent preview tools can remain attached to the installed T3 Code environment instead of the worktree host. That exact claim requires preview automation routed to the isolated dev desktop. Primary files: @@ -62,17 +60,19 @@ Primary files: - `packages/contracts/src/ipc.ts` - `scripts/dev-runner.ts` -Focused regression coverage lives in `scripts/dev-runner.test.ts`, `apps/desktop/src/app/DesktopAppIdentity.test.ts`, `apps/desktop/src/app/DesktopEnvironment.test.ts`, `apps/desktop/src/preview/Manager.test.ts`, `apps/server/src/mcp/McpHttpServer.test.ts`, `apps/web/src/browser/browserSurfaceStore.test.ts`, `apps/web/src/browser/hostedBrowserWebviewStyle.test.ts`, `apps/web/src/components/auth/PairingRouteSurface.logic.test.ts`, `apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationPresentation.test.ts`, `apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts`, `packages/contracts/src/ipc.test.ts`, and `packages/contracts/src/preview.test.ts`. +Focused regression coverage lives in `scripts/dev-runner.test.ts`, `apps/desktop/src/app/DesktopAppIdentity.test.ts`, `apps/desktop/src/app/DesktopEnvironment.test.ts`, `apps/desktop/src/preview/Manager.test.ts`, `apps/server/src/mcp/McpHttpServer.test.ts`, `apps/web/src/browser/browserSurfaceStore.test.ts`, `apps/web/src/browser/hostedBrowserWebviewStyle.test.ts`, `apps/web/src/browser/browserViewportActions.test.ts`, `apps/web/src/browser/browserViewportLayout.test.ts`, `apps/web/src/browser/previewRuntimeTabId.test.ts`, `apps/web/src/components/auth/PairingRouteSurface.logic.test.ts`, `apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationPresentation.test.ts`, `apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts`, `apps/web/src/components/preview/previewNavigationReadiness.test.ts`, `apps/web/src/components/preview/previewViewportRollback.test.ts`, `packages/contracts/src/ipc.test.ts`, and `packages/contracts/src/preview.test.ts`. ```sh -vp test run scripts/dev-runner.test.ts apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/web/src/browser/browserSurfaceStore.test.ts apps/web/src/browser/hostedBrowserWebviewStyle.test.ts apps/web/src/components/auth/PairingRouteSurface.logic.test.ts apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts apps/web/src/components/preview/previewAutomationPresentation.test.ts apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts packages/contracts/src/ipc.test.ts packages/contracts/src/preview.test.ts +vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.test.ts apps/desktop/src/app/DesktopEnvironment.test.ts apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/web/src/browser/browserSurfaceStore.test.ts apps/web/src/browser/hostedBrowserWebviewStyle.test.ts apps/web/src/browser/browserViewportActions.test.ts apps/web/src/browser/browserViewportLayout.test.ts apps/web/src/browser/previewRuntimeTabId.test.ts apps/web/src/components/auth/PairingRouteSurface.logic.test.ts apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts apps/web/src/components/preview/previewAutomationPresentation.test.ts apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts apps/web/src/components/preview/previewNavigationReadiness.test.ts apps/web/src/components/preview/previewViewportRollback.test.ts packages/contracts/src/ipc.test.ts packages/contracts/src/preview.test.ts ``` -Verification completed for the current uncommitted follow-up: +Current verification: -- `vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.test.ts apps/desktop/src/app/DesktopEnvironment.test.ts` passed all 77 tests. -- `vp run --filter @t3tools/desktop typecheck` and `vp run --filter ./scripts typecheck` completed without type errors. The desktop check still printed two Effect suggestions in unchanged files. -- A worktree `dev:desktop` instance using the isolated user-data override paired successfully and stayed usable through repeated renderer/CDP inspection and raster capture. An archive worktree client loaded its seeded project against that host, and the prior IndexedDB `VersionError` and host-interface disappearance did not recur in those exercised flows. +- The focused command above passed all 218 tests. +- The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, and `previewAutomationRequestConsumer`) passed all 37 tests with explicit runtime guest identity. +- Desktop, server, contracts, and scripts typechecks completed without type errors. The web typecheck currently reaches only three `RegistryContext` typing errors that are also present on `upstream/main`. +- An isolated worktree `dev:desktop` using the user-data override paired successfully and stayed usable through repeated renderer/CDP inspection and raster capture. An archive worktree client loaded its seeded project against that host without the prior IndexedDB `VersionError` or host-interface disappearance. +- An isolated web client on ports `5744`/`13784` completed first-navigation pairing, loaded the seeded Preview Reliability thread, and rendered the right-panel surface chooser. Its non-Electron Browser surface was unavailable as described under current limitations. ## Development Ports From facd17fd356deb95d8e69411faadc159a7d397e3 Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Tue, 28 Jul 2026 15:52:31 +0100 Subject: [PATCH 25/43] Abort preview visibility waits on runtime changes --- apps/web/src/components/preview/PreviewAutomationHosts.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index a1167083d99..58fa8e7d484 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -133,6 +133,10 @@ const waitForBrowserSurfaceVisibility = async ( const requiredStableMs = Math.min(100, Math.max(0, timeoutMs - 50)); let presentedSince: number | null = null; while (Date.now() <= deadline) { + assertPreviewRuntimeCurrent(threadRef, tabId, runtimeTabId, { + operation: "open", + requestId, + }); const now = Date.now(); if (isPreviewAutomationTabPresented(threadRef, tabId, runtimeTabId)) { presentedSince ??= now; From 168990f5621aba94ad0d3c4498b8e7a4265cce2e Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Tue, 28 Jul 2026 16:03:10 +0100 Subject: [PATCH 26/43] Harden preview runtime replacement handling - Reject stale open and snapshot presentation waits immediately - Use named server and runtime presentation targets - Cover runtime replacement behavior and document the boundary --- BRANCH_DETAILS.md | 6 +- .../preview/PreviewAutomationHosts.tsx | 69 ++---- .../previewAutomationPresentation.test.ts | 214 +++++++++++++----- .../preview/previewAutomationPresentation.ts | 137 +++++++---- 4 files changed, 273 insertions(+), 153 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index b70e5f8a36d..e1ca1416bfc 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -21,11 +21,11 @@ Expected behavior: - Click, type, and wait operations propagate their caller-supplied timeout into the desktop control-session boundary. Operations without a caller timeout retain the bounded desktop default. - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. A persisted non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. -- Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. +- Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the server epoch replaces the runtime guest while staging is pending, the snapshot fails immediately with `PreviewAutomationTargetUnavailableError` instead of waiting on the stale staging marker. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. -- A newly created preview tab applies its server snapshot and assigned tab id, applies the upstream 1280×800 automation viewport when the server snapshot still uses `fill`, initiates any requested selection, and acknowledges server-side creation without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across route hydration or session reconciliation instead of accepting one transient visible frame. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for `open`. +- A newly created preview tab applies its server snapshot and assigned tab id, applies the upstream 1280×800 automation viewport when the server snapshot still uses `fill`, initiates any requested selection, and acknowledges server-side creation without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across same-server route hydration or session reconciliation instead of accepting one transient visible frame. A server-epoch change denotes a replacement runtime guest and aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the new guest. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for `open`. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. - `dev:desktop` derives `T3CODE_DESKTOP_USER_DATA_DIR=/userdata/electron` whenever the runner has an explicit base directory. Desktop configuration resolves that override to an absolute path, and app identity uses it before legacy migration or the normal Electron user-data default. This keeps an isolated worktree dev desktop from reusing an installed or earlier development profile whose incompatible IndexedDB schema can prevent the renderer from starting. Packaged/default startup remains unchanged when no override is supplied. @@ -69,7 +69,7 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.t Current verification: - The focused command above passed all 218 tests. -- The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, and `previewAutomationRequestConsumer`) passed all 37 tests with explicit runtime guest identity. +- The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 40 tests with explicit runtime guest identity, including immediate open and snapshot rejection after a server-epoch replacement. - Desktop, server, contracts, and scripts typechecks completed without type errors. The web typecheck currently reaches only three `RegistryContext` typing errors that are also present on `upstream/main`. - An isolated worktree `dev:desktop` using the user-data override paired successfully and stayed usable through repeated renderer/CDP inspection and raster capture. An archive worktree client loaded its seeded project against that host without the prior IndexedDB `VersionError` or host-interface disappearance. - An isolated web client on ports `5744`/`13784` completed first-navigation pairing, loaded the seeded Preview Reliability thread, and rendered the right-panel surface chooser. Its non-Electron Browser surface was unavailable as described under current limitations. diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 58fa8e7d484..eb20c7dea2d 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -47,9 +47,8 @@ import { useAtomCommand } from "~/state/use-atom-command"; import { previewBridge } from "./previewBridge"; import { - isPreviewAutomationTabPresented, - readPreviewAutomationPresentationDiagnostics, revealPreviewAutomationTab, + waitForBrowserSurfaceVisibility, withPreviewAutomationBackgroundPresentation, } from "./previewAutomationPresentation"; import { @@ -58,7 +57,6 @@ import { PreviewAutomationOverlayTimeoutError, PreviewAutomationRecordingNotActiveError, PreviewAutomationTargetUnavailableError, - PreviewAutomationVisibilityTimeoutError, PreviewAutomationViewportTimeoutError, } from "./previewAutomationErrors"; import { @@ -122,45 +120,6 @@ const waitForDesktopOverlay = async ( }); }; -const waitForBrowserSurfaceVisibility = async ( - threadRef: ScopedThreadRef, - requestId: string, - tabId: string, - runtimeTabId: string, - timeoutMs: number, -): Promise => { - const deadline = Date.now() + timeoutMs; - const requiredStableMs = Math.min(100, Math.max(0, timeoutMs - 50)); - let presentedSince: number | null = null; - while (Date.now() <= deadline) { - assertPreviewRuntimeCurrent(threadRef, tabId, runtimeTabId, { - operation: "open", - requestId, - }); - const now = Date.now(); - if (isPreviewAutomationTabPresented(threadRef, tabId, runtimeTabId)) { - presentedSince ??= now; - // Require the selection to survive multiple presentation updates. A - // single transient `visible` frame can otherwise make open acknowledge - // just before routing or panel reconciliation unmounts the surface. - if (now - presentedSince >= requiredStableMs) return; - } else { - presentedSince = null; - // Session reconciliation and route hydration can race a cold open. - // Reassert the explicit show request only while that request is pending. - revealPreviewAutomationTab(threadRef, tabId); - } - await new Promise((resolve) => window.setTimeout(resolve, 50)); - } - throw new PreviewAutomationVisibilityTimeoutError({ - requestId, - environmentId: threadRef.environmentId, - threadId: threadRef.threadId, - tabId, - timeoutMs, - ...readPreviewAutomationPresentationDiagnostics(threadRef, tabId, runtimeTabId), - }); -}; interface ExecutablePreviewWebview extends Element { readonly executeJavaScript: (code: string, userGesture?: boolean) => Promise; } @@ -533,13 +492,13 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) ); } if (waitPolicy?.waitForVisibility) { - await waitForBrowserSurfaceVisibility( + await waitForBrowserSurfaceVisibility({ threadRef, - request.requestId, - activeTabId, - activeRuntimeTabId, - remainingOperationBudget(), - ); + requestId: request.requestId, + tabId: activeTabId, + runtimeTabId: activeRuntimeTabId, + timeoutMs: remainingOperationBudget(), + }); } return await currentStatus(threadRef, activeTabId); } @@ -657,19 +616,19 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) case "snapshot": { const ready = await requireReadyTab(); const presentationTimeoutMs = remainingOperationBudget(); - return await withPreviewAutomationBackgroundPresentation( + return await withPreviewAutomationBackgroundPresentation({ threadRef, - request.requestId, - ready.tabId, - ready.runtimeTabId, - presentationTimeoutMs, - async (background) => + requestId: request.requestId, + tabId: ready.tabId, + runtimeTabId: ready.runtimeTabId, + timeoutMs: presentationTimeoutMs, + use: async (background) => await ready.bridge.automation.snapshot( ready.runtimeTabId, background, remainingOperationBudget(), ), - ); + }); } case "click": { const ready = await requireReadyTab(); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index 2e8c63d7712..afcffc4bc4f 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -7,6 +7,7 @@ import { previewRuntimeTabId } from "~/browser/previewRuntimeTabId"; import { applyPreviewServerSnapshot, readThreadPreviewState, + reconcilePreviewServerSessions, resetPreviewStateForTests, } from "~/previewStateStore"; import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; @@ -16,9 +17,11 @@ import { isPreviewAutomationTabPresented, readPreviewAutomationPresentationDiagnostics, revealPreviewAutomationTab, + waitForBrowserSurfaceVisibility, withPreviewAutomationBackgroundPresentation, waitForPreviewAutomationBackgroundPresentation, } from "./previewAutomationPresentation"; +import { PreviewAutomationTargetUnavailableError } from "./previewAutomationErrors"; const threadRef = scopeThreadRef(EnvironmentId.make("environment-1"), ThreadId.make("thread-1")); @@ -31,6 +34,17 @@ const snapshot = (tabId: string, updatedAt: string): PreviewSessionSnapshot => ( updatedAt, }); +const presentationTarget = (tabId: string, runtimeTabId: string) => ({ + threadRef, + tabId, + runtimeTabId, +}); + +const addRuntimeTab = (tabId: string): string => { + applyPreviewServerSnapshot(threadRef, snapshot(tabId, "2026-07-25T00:00:00.000Z")); + return previewRuntimeTabId(threadRef, readThreadPreviewState(threadRef).serverEpoch, tabId); +}; + describe("preview automation presentation", () => { beforeEach(() => { resetPreviewStateForTests(); @@ -56,9 +70,11 @@ describe("preview automation presentation", () => { ).toMatchObject({ tabId: "tab-1", }); - expect(isPreviewAutomationTabPresented(threadRef, "tab-1", tabOneRuntimeId)).toBe(false); + expect(isPreviewAutomationTabPresented(presentationTarget("tab-1", tabOneRuntimeId))).toBe( + false, + ); expect( - readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1", tabOneRuntimeId), + readPreviewAutomationPresentationDiagnostics(presentationTarget("tab-1", tabOneRuntimeId)), ).toEqual({ activeSurfaceKind: "inline-preview", activeSurfaceId: "mini-player:tab-1", @@ -72,9 +88,11 @@ describe("preview automation presentation", () => { const serverIdSurface = acquireBrowserSurface("tab-1"); serverIdSurface.present({ x: 0, y: 0, width: 800, height: 600 }, true); - expect(isPreviewAutomationTabPresented(threadRef, "tab-1", tabOneRuntimeId)).toBe(false); + expect(isPreviewAutomationTabPresented(presentationTarget("tab-1", tabOneRuntimeId))).toBe( + false, + ); expect( - readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1", tabOneRuntimeId), + readPreviewAutomationPresentationDiagnostics(presentationTarget("tab-1", tabOneRuntimeId)), ).toMatchObject({ surfaceRegistered: false, presentationRectAvailable: false, @@ -84,18 +102,22 @@ describe("preview automation presentation", () => { const surface = acquireBrowserSurface(tabOneRuntimeId); surface.present({ x: 0, y: 0, width: 800, height: 600 }, true); - expect(isPreviewAutomationTabPresented(threadRef, "tab-1", tabOneRuntimeId)).toBe(true); + expect(isPreviewAutomationTabPresented(presentationTarget("tab-1", tabOneRuntimeId))).toBe( + true, + ); expect( - readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1", tabOneRuntimeId), + readPreviewAutomationPresentationDiagnostics(presentationTarget("tab-1", tabOneRuntimeId)), ).toMatchObject({ surfaceRegistered: true, presentationRectAvailable: true, }); usePreviewMiniPlayerStore.getState().open(threadRef, "tab-2"); - expect(isPreviewAutomationTabPresented(threadRef, "tab-1", tabOneRuntimeId)).toBe(false); + expect(isPreviewAutomationTabPresented(presentationTarget("tab-1", tabOneRuntimeId))).toBe( + false, + ); expect( - readPreviewAutomationPresentationDiagnostics(threadRef, "tab-1", tabOneRuntimeId), + readPreviewAutomationPresentationDiagnostics(presentationTarget("tab-1", tabOneRuntimeId)), ).toEqual({ activeSurfaceKind: "inline-preview", activeSurfaceId: "mini-player:tab-2", @@ -111,7 +133,9 @@ describe("preview automation presentation", () => { it("reports presentation precedence and hidden retained panel state", () => { expect( - readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested", "runtime-requested"), + readPreviewAutomationPresentationDiagnostics( + presentationTarget("tab-requested", "runtime-requested"), + ), ).toEqual({ activeSurfaceKind: "none", activeSurfaceId: null, @@ -127,7 +151,9 @@ describe("preview automation presentation", () => { usePreviewMiniPlayerStore.getState().open(threadRef, "tab-inline"); expect( - readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested", "runtime-requested"), + readPreviewAutomationPresentationDiagnostics( + presentationTarget("tab-requested", "runtime-requested"), + ), ).toEqual({ activeSurfaceKind: "inline-preview", activeSurfaceId: "mini-player:tab-inline", @@ -143,7 +169,9 @@ describe("preview automation presentation", () => { useRightPanelStore.getState().close(threadRef); expect( - readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested", "runtime-requested"), + readPreviewAutomationPresentationDiagnostics( + presentationTarget("tab-requested", "runtime-requested"), + ), ).toEqual({ activeSurfaceKind: "none", activeSurfaceId: null, @@ -160,7 +188,9 @@ describe("preview automation presentation", () => { useRightPanelStore.getState().openBrowser(threadRef, "tab-panel"); expect( - readPreviewAutomationPresentationDiagnostics(threadRef, "tab-requested", "runtime-requested"), + readPreviewAutomationPresentationDiagnostics( + presentationTarget("tab-requested", "runtime-requested"), + ), ).toEqual({ activeSurfaceKind: "right-panel", activeSurfaceId: "browser:tab-panel", @@ -175,6 +205,7 @@ describe("preview automation presentation", () => { it("uses the operation budget when background staging does not render", async () => { vi.useFakeTimers(); + const runtimeTabId = addRuntimeTab("tab-background"); vi.stubGlobal("document", { querySelectorAll: () => [], }); @@ -186,7 +217,7 @@ describe("preview automation presentation", () => { threadRef, requestId: "request-background", tabId: "tab-background", - runtimeTabId: "runtime-background", + runtimeTabId, timeoutMs: 40, }); const rejection = expect(presentation).rejects.toMatchObject({ @@ -204,22 +235,104 @@ describe("preview automation presentation", () => { } }); + it("rejects an open visibility wait when the runtime guest is replaced", async () => { + vi.useFakeTimers(); + vi.stubGlobal("window", { + setTimeout, + }); + const tabId = "tab-open"; + const serverSnapshot = snapshot(tabId, "2026-07-25T00:00:00.000Z"); + reconcilePreviewServerSessions(threadRef, { + sessions: [serverSnapshot], + serverEpoch: "epoch-1", + revision: 1, + }); + const runtimeTabId = previewRuntimeTabId(threadRef, "epoch-1", tabId); + try { + const visibility = waitForBrowserSurfaceVisibility({ + threadRef, + requestId: "request-open", + tabId, + runtimeTabId, + timeoutMs: 500, + }); + const rejection = expect(visibility).rejects.toBeInstanceOf( + PreviewAutomationTargetUnavailableError, + ); + + reconcilePreviewServerSessions(threadRef, { + sessions: [serverSnapshot], + serverEpoch: "epoch-2", + revision: 0, + }); + await vi.advanceTimersByTimeAsync(50); + + await rejection; + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); + + it("rejects background staging when the runtime guest is replaced", async () => { + vi.useFakeTimers(); + vi.stubGlobal("document", { + querySelectorAll: () => [], + }); + vi.stubGlobal("window", { + setTimeout, + }); + const tabId = "tab-background"; + const serverSnapshot = snapshot(tabId, "2026-07-25T00:00:00.000Z"); + reconcilePreviewServerSessions(threadRef, { + sessions: [serverSnapshot], + serverEpoch: "epoch-1", + revision: 1, + }); + const runtimeTabId = previewRuntimeTabId(threadRef, "epoch-1", tabId); + try { + const presentation = waitForPreviewAutomationBackgroundPresentation({ + threadRef, + requestId: "request-background", + tabId, + runtimeTabId, + timeoutMs: 500, + }); + const rejection = expect(presentation).rejects.toBeInstanceOf( + PreviewAutomationTargetUnavailableError, + ); + + reconcilePreviewServerSessions(threadRef, { + sessions: [serverSnapshot], + serverEpoch: "epoch-2", + revision: 0, + }); + await vi.advanceTimersByTimeAsync(16); + + await rejection; + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); + it("accepts a tab that becomes foregrounded while background staging renders", async () => { vi.useFakeTimers(); + const runtimeTabId = addRuntimeTab("tab-foregrounded"); vi.stubGlobal("document", { querySelectorAll: () => [], }); vi.stubGlobal("window", { setTimeout, }); - const surface = acquireBrowserSurface("tab-foregrounded"); + const surface = acquireBrowserSurface(runtimeTabId); try { revealPreviewAutomationTab(threadRef, "tab-foregrounded"); const presentation = waitForPreviewAutomationBackgroundPresentation({ threadRef, requestId: "request-foregrounded", tabId: "tab-foregrounded", - runtimeTabId: "tab-foregrounded", + runtimeTabId, timeoutMs: 40, }); @@ -235,11 +348,12 @@ describe("preview automation presentation", () => { }); it("stages a visible surface when another inline preview surface is selected", async () => { + const runtimeTabId = addRuntimeTab("tab-background"); vi.stubGlobal("document", { querySelectorAll: () => [ { dataset: { - previewViewport: "tab-background", + previewViewport: runtimeTabId, previewBackgroundCapture: "true", }, offsetWidth: 800, @@ -252,28 +366,28 @@ describe("preview automation presentation", () => { return 1; }, }); - const surface = acquireBrowserSurface("tab-background"); + const surface = acquireBrowserSurface(runtimeTabId); try { surface.present({ x: 0, y: 0, width: 800, height: 600 }, true); revealPreviewAutomationTab(threadRef, "tab-foreground"); - const background = await withPreviewAutomationBackgroundPresentation( + const background = await withPreviewAutomationBackgroundPresentation({ threadRef, - "request-background", - "tab-background", - "tab-background", - 40, - async (isBackground) => { + requestId: "request-background", + tabId: "tab-background", + runtimeTabId, + timeoutMs: 40, + use: async (isBackground) => { expect( - useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId[runtimeTabId], ).toBe(1); return isBackground; }, - ); + }); expect(background).toBe(true); expect( - useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId[runtimeTabId], ).toBeUndefined(); } finally { surface.release(); @@ -283,12 +397,13 @@ describe("preview automation presentation", () => { it("falls back to frame timers when compositor animation frames remain paused", async () => { vi.useFakeTimers(); + const runtimeTabId = addRuntimeTab("tab-background"); const cancelAnimationFrame = vi.fn(); vi.stubGlobal("document", { querySelectorAll: () => [ { dataset: { - previewViewport: "tab-background", + previewViewport: runtimeTabId, previewBackgroundCapture: "true", }, offsetWidth: 800, @@ -301,17 +416,15 @@ describe("preview automation presentation", () => { }); const use = vi.fn(async (background: boolean) => background); try { - const operation = withPreviewAutomationBackgroundPresentation( + const operation = withPreviewAutomationBackgroundPresentation({ threadRef, - "request-paused-frame", - "tab-background", - "tab-background", - 40, + requestId: "request-paused-frame", + tabId: "tab-background", + runtimeTabId, + timeoutMs: 40, use, - ); - expect( - useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], - ).toBe(1); + }); + expect(useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId[runtimeTabId]).toBe(1); await vi.advanceTimersByTimeAsync(32); await expect(operation).resolves.toBe(true); @@ -321,7 +434,7 @@ describe("preview automation presentation", () => { expect(cancelAnimationFrame).toHaveBeenNthCalledWith(1, 1); expect(cancelAnimationFrame).toHaveBeenNthCalledWith(2, 1); expect( - useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId[runtimeTabId], ).toBeUndefined(); } finally { vi.unstubAllGlobals(); @@ -331,11 +444,12 @@ describe("preview automation presentation", () => { it("retains a background capture lease until a timed-out operation settles", async () => { vi.useFakeTimers(); + const runtimeTabId = addRuntimeTab("tab-background"); vi.stubGlobal("document", { querySelectorAll: () => [ { dataset: { - previewViewport: "tab-background", + previewViewport: runtimeTabId, previewBackgroundCapture: "true", }, offsetWidth: 800, @@ -353,14 +467,14 @@ describe("preview automation presentation", () => { const stalledOperation = new Promise((resolve) => { settleOperation = resolve; }); - const operation = withPreviewAutomationBackgroundPresentation( + const operation = withPreviewAutomationBackgroundPresentation({ threadRef, - "request-stalled", - "tab-background", - "tab-background", - 40, - () => stalledOperation, - ); + requestId: "request-stalled", + tabId: "tab-background", + runtimeTabId, + timeoutMs: 40, + use: () => stalledOperation, + }); const rejection = expect(operation).rejects.toMatchObject({ _tag: "PreviewAutomationBackgroundPresentationTimeoutError", requestId: "request-stalled", @@ -368,21 +482,17 @@ describe("preview automation presentation", () => { timeoutMs: 40, }); await Promise.resolve(); - expect( - useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], - ).toBe(1); + expect(useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId[runtimeTabId]).toBe(1); await vi.advanceTimersByTimeAsync(40); await rejection; - expect( - useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], - ).toBe(1); + expect(useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId[runtimeTabId]).toBe(1); settleOperation(); await stalledOperation; await Promise.resolve(); expect( - useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId["tab-background"], + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId[runtimeTabId], ).toBeUndefined(); } finally { vi.unstubAllGlobals(); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index 2708767813d..bbc29fc8532 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -8,17 +8,34 @@ import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/prev import { setActivePreviewTab } from "~/previewStateStore"; import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelStore"; -import { PreviewAutomationBackgroundPresentationTimeoutError } from "./previewAutomationErrors"; -import type { PreviewAutomationVisibilityTimeoutError } from "./previewAutomationErrors"; +import { + PreviewAutomationBackgroundPresentationTimeoutError, + PreviewAutomationVisibilityTimeoutError, +} from "./previewAutomationErrors"; +import { assertPreviewRuntimeCurrent } from "./previewNavigationReadiness"; -interface PreviewAutomationBackgroundPresentationInput { +interface PreviewAutomationPresentationTarget { readonly threadRef: ScopedThreadRef; - readonly requestId: string; readonly tabId: string; readonly runtimeTabId: string; +} + +interface PreviewAutomationBackgroundPresentationInput extends PreviewAutomationPresentationTarget { + readonly requestId: string; + readonly timeoutMs: number; +} + +interface PreviewAutomationVisibilityInput extends PreviewAutomationPresentationTarget { + readonly requestId: string; readonly timeoutMs: number; } +interface PreviewAutomationBackgroundPresentationUseInput< + A, +> extends PreviewAutomationBackgroundPresentationInput { + readonly use: (background: boolean) => Promise; +} + type PreviewAutomationPresentationDiagnostics = Required< Pick< PreviewAutomationVisibilityTimeoutError, @@ -79,30 +96,23 @@ export function revealPreviewAutomationTab(ref: ScopedThreadRef, tabId: string): usePreviewMiniPlayerStore.getState().open(ref, tabId); } -function readPreviewAutomationPresentation( - ref: ScopedThreadRef, - tabId: string, - runtimeTabId: string, -) { - const panel = selectThreadRightPanelState(useRightPanelStore.getState().byThreadKey, ref); +function readPreviewAutomationPresentation(input: PreviewAutomationPresentationTarget) { + const panel = selectThreadRightPanelState( + useRightPanelStore.getState().byThreadKey, + input.threadRef, + ); const miniPlayer = selectThreadPreviewMiniPlayer( usePreviewMiniPlayerStore.getState().byThreadKey, - ref, + input.threadRef, ); - const presentation = useBrowserSurfaceStore.getState().byTabId[runtimeTabId]; + const presentation = useBrowserSurfaceStore.getState().byTabId[input.runtimeTabId]; return { panel, miniPlayer, presentation }; } export function readPreviewAutomationPresentationDiagnostics( - ref: ScopedThreadRef, - tabId: string, - runtimeTabId: string, + input: PreviewAutomationPresentationTarget, ): PreviewAutomationPresentationDiagnostics { - const { panel, miniPlayer, presentation } = readPreviewAutomationPresentation( - ref, - tabId, - runtimeTabId, - ); + const { panel, miniPlayer, presentation } = readPreviewAutomationPresentation(input); const activeSurfaceKind = miniPlayer !== null ? ("inline-preview" as const) @@ -127,27 +137,62 @@ export function readPreviewAutomationPresentationDiagnostics( } export function isPreviewAutomationTabPresented( - ref: ScopedThreadRef, - tabId: string, - runtimeTabId: string, + input: PreviewAutomationPresentationTarget, ): boolean { - const { panel, miniPlayer, presentation } = readPreviewAutomationPresentation( - ref, - tabId, - runtimeTabId, - ); + const { panel, miniPlayer, presentation } = readPreviewAutomationPresentation(input); const requestedSurfaceIsActive = - (panel.isOpen && panel.activeSurfaceId === `browser:${tabId}`) || miniPlayer?.tabId === tabId; + (panel.isOpen && panel.activeSurfaceId === `browser:${input.tabId}`) || + miniPlayer?.tabId === input.tabId; return requestedSurfaceIsActive && (presentation?.visible ?? false); } +export async function waitForBrowserSurfaceVisibility( + input: PreviewAutomationVisibilityInput, +): Promise { + const deadline = Date.now() + input.timeoutMs; + const requiredStableMs = Math.min(100, Math.max(0, input.timeoutMs - 50)); + let presentedSince: number | null = null; + while (Date.now() <= deadline) { + assertPreviewRuntimeCurrent(input.threadRef, input.tabId, input.runtimeTabId, { + operation: "open", + requestId: input.requestId, + }); + const now = Date.now(); + if (isPreviewAutomationTabPresented(input)) { + presentedSince ??= now; + // Require the selection to survive multiple presentation updates. A + // single transient `visible` frame can otherwise make open acknowledge + // just before routing or panel reconciliation unmounts the surface. + if (now - presentedSince >= requiredStableMs) return; + } else { + presentedSince = null; + // Same-server reconciliation and route hydration can race a cold open. + // Reassert the explicit show request only while that request is pending. + revealPreviewAutomationTab(input.threadRef, input.tabId); + } + await new Promise((resolve) => window.setTimeout(resolve, 50)); + } + throw new PreviewAutomationVisibilityTimeoutError({ + requestId: input.requestId, + environmentId: input.threadRef.environmentId, + threadId: input.threadRef.threadId, + tabId: input.tabId, + timeoutMs: input.timeoutMs, + ...readPreviewAutomationPresentationDiagnostics(input), + }); +} + export async function waitForPreviewAutomationBackgroundPresentation( input: PreviewAutomationBackgroundPresentationInput, ): Promise { const deadline = Date.now() + input.timeoutMs; const timeoutError = () => backgroundPresentationTimeoutError(input); while (true) { - if (isPreviewAutomationTabPresented(input.threadRef, input.tabId, input.runtimeTabId)) { + assertPreviewRuntimeCurrent(input.threadRef, input.tabId, input.runtimeTabId, { + operation: "snapshot", + requestId: input.requestId, + }); + if (isPreviewAutomationTabPresented(input)) { return; } @@ -166,6 +211,10 @@ export async function waitForPreviewAutomationBackgroundPresentation( void wrapper.offsetWidth; await waitForPreviewAutomationCompositorFrame(deadline, timeoutError); await waitForPreviewAutomationCompositorFrame(deadline, timeoutError); + assertPreviewRuntimeCurrent(input.threadRef, input.tabId, input.runtimeTabId, { + operation: "snapshot", + requestId: input.requestId, + }); return; } @@ -178,20 +227,18 @@ export async function waitForPreviewAutomationBackgroundPresentation( } export async function withPreviewAutomationBackgroundPresentation( - threadRef: ScopedThreadRef, - requestId: string, - tabId: string, - runtimeTabId: string, - timeoutMs: number, - use: (background: boolean) => Promise, + input: PreviewAutomationBackgroundPresentationUseInput, ): Promise { - const background = !isPreviewAutomationTabPresented(threadRef, tabId, runtimeTabId); - if (!background) return await use(false); + assertPreviewRuntimeCurrent(input.threadRef, input.tabId, input.runtimeTabId, { + operation: "snapshot", + requestId: input.requestId, + }); + const background = !isPreviewAutomationTabPresented(input); + if (!background) return await input.use(false); - const input = { threadRef, requestId, tabId, runtimeTabId, timeoutMs }; const timeoutError = () => backgroundPresentationTimeoutError(input); - const deadline = Date.now() + timeoutMs; - const releaseCapture = acquireBrowserSurfaceBackgroundCapture(runtimeTabId); + const deadline = Date.now() + input.timeoutMs; + const releaseCapture = acquireBrowserSurfaceBackgroundCapture(input.runtimeTabId); let captureStarted = false; let timer: ReturnType | undefined; @@ -200,8 +247,12 @@ export async function withPreviewAutomationBackgroundPresentation( const remainingMs = deadline - Date.now(); if (remainingMs <= 0) throw timeoutError(); - const stillBackground = !isPreviewAutomationTabPresented(threadRef, tabId, runtimeTabId); - const capture = use(stillBackground); + assertPreviewRuntimeCurrent(input.threadRef, input.tabId, input.runtimeTabId, { + operation: "snapshot", + requestId: input.requestId, + }); + const stillBackground = !isPreviewAutomationTabPresented(input); + const capture = input.use(stillBackground); captureStarted = true; const operation = capture.finally(releaseCapture); const captureDeadline = new Promise((_resolve, reject) => { From 75f2414442db84deae41631f25b8d24f8bdc3b03 Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Tue, 28 Jul 2026 17:46:53 +0100 Subject: [PATCH 27/43] Harden preview deadline budgeting - Keep response grace monotonic for short automation timeouts - Reserve bounded screenshot fallback and settlement time - Clamp browser visibility polling to the remaining deadline --- BRANCH_DETAILS.md | 10 +-- apps/desktop/src/preview/Manager.test.ts | 38 +++++--- apps/desktop/src/preview/Manager.ts | 88 ++++++++++++++----- .../previewAutomationPresentation.test.ts | 29 ++++++ .../preview/previewAutomationPresentation.ts | 4 +- .../previewAutomationRequestConsumer.test.ts | 13 +++ .../previewAutomationRequestConsumer.ts | 2 +- 7 files changed, 145 insertions(+), 39 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index e1ca1416bfc..27ad1a0c15f 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -17,14 +17,14 @@ This branch does not add a second recording/PiP capture lifecycle or another hid Expected behavior: -- Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session when initialization or an acquired permit may have left a CDP command pending. A request that times out while queued behind another action does not detach that action's shared debugger session. +- Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout without making the execution budget shrink when the caller increases a short timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session when initialization or an acquired permit may have left a CDP command pending. A request that times out while queued behind another action does not detach that action's shared debugger session. - Click, type, and wait operations propagate their caller-supplied timeout into the desktop control-session boundary. Operations without a caller timeout retain the bounded desktop default. -- Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. +- Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Primary and fallback screenshot waits are clamped to the remaining control-session deadline, with budget reserved for fallback and result settlement, so a tight caller deadline can still return semantic data instead of being preempted by the outer session timeout. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. A persisted non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. - Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the server epoch replaces the runtime guest while staging is pending, the snapshot fails immediately with `PreviewAutomationTargetUnavailableError` instead of waiting on the stale staging marker. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. -- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget instead of being consumed by fixed grace deductions. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and their stable-presentation dwell contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. +- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget, and the transition into grace reservation remains monotonic as requested timeouts increase. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and both visibility polling and stable-presentation dwell contract to fit short deadlines instead of overshooting them or requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. - A newly created preview tab applies its server snapshot and assigned tab id, applies the upstream 1280×800 automation viewport when the server snapshot still uses `fill`, initiates any requested selection, and acknowledges server-side creation without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across same-server route hydration or session reconciliation instead of accepting one transient visible frame. A server-epoch change denotes a replacement runtime guest and aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the new guest. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for `open`. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. @@ -68,8 +68,8 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.t Current verification: -- The focused command above passed all 218 tests. -- The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 40 tests with explicit runtime guest identity, including immediate open and snapshot rejection after a server-epoch replacement. +- The focused command above passed all 223 tests. +- The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 42 tests with explicit runtime guest identity, including immediate open, monotonic short-deadline handling, deadline-clamped visibility polling, and snapshot rejection after a server-epoch replacement. - Desktop, server, contracts, and scripts typechecks completed without type errors. The web typecheck currently reaches only three `RegistryContext` typing errors that are also present on `upstream/main`. - An isolated worktree `dev:desktop` using the user-data override paired successfully and stayed usable through repeated renderer/CDP inspection and raster capture. An archive worktree client loaded its seeded project against that host without the prior IndexedDB `VersionError` or host-interface disappearance. - An isolated web client on ports `5744`/`13784` completed first-navigation pairing, loaded the seeded Preview Reliability thread, and rendered the right-panel surface chooser. Its non-Electron Browser surface was unavailable as described under current limitations. diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index c482bf9fd31..5810051d339 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -36,6 +36,24 @@ describe("fitPictureInPictureContentSize", () => { }); }); +describe("automationExecutionBudget", () => { + it("keeps short execution budgets monotonic while reserving response grace", () => { + expect(PreviewManager.automationExecutionBudget(100)).toBe(100); + expect(PreviewManager.automationExecutionBudget(500)).toBe(500); + expect(PreviewManager.automationExecutionBudget(501)).toBe(500); + expect(PreviewManager.automationExecutionBudget(750)).toBe(500); + expect(PreviewManager.automationExecutionBudget(751)).toBe(501); + expect(PreviewManager.automationExecutionBudget(1_000)).toBe(750); + + const budgets = Array.from({ length: 1_001 }, (_, timeoutMs) => + PreviewManager.automationExecutionBudget(timeoutMs), + ); + expect(budgets.every((budget, index) => index === 0 || budget >= budgets[index - 1]!)).toBe( + true, + ); + }); +}); + const { bridgeAttach, bridgeDestroy, @@ -468,20 +486,20 @@ describe("PreviewManager", () => { expect(recovered.screenshot).toMatchObject({ width: 640, height: 360 }); expect(attach).toHaveBeenCalledTimes(2); + capturePage.mockClear(); captureMode = "timeout"; + capturePage.mockImplementationOnce(async () => await new Promise(() => undefined)); const callerBoundCapture = yield* manager .automationSnapshot("tab_snapshot", false, 1_000) .pipe(Effect.forkChild({ startImmediately: true })); - yield* TestClock.adjust(750); - const callerBoundExit = yield* Effect.exit(Fiber.join(callerBoundCapture)); - expect(Exit.isFailure(callerBoundExit)).toBe(true); - if (Exit.isFailure(callerBoundExit)) { - expect(Option.getOrThrow(Cause.findErrorOption(callerBoundExit.cause))).toMatchObject({ - _tag: "PreviewAutomationTimeoutError", - tabId: "tab_snapshot", - timeoutMs: 1_000, - }); - } + yield* TestClock.adjust(725); + const callerBoundResult = yield* Fiber.join(callerBoundCapture); + expect(callerBoundResult).toMatchObject({ + url: "https://example.com/", + visibleText: "Example body", + screenshot: null, + }); + expect(capturePage).toHaveBeenCalledOnce(); expect(detach).toHaveBeenCalledTimes(2); }), ), diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 6e1521be8b6..f3834126af7 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -103,10 +103,15 @@ const AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS = 250; const AUTOMATION_SCREENSHOT_TIMEOUT_MS = 5_000; const AUTOMATION_BACKGROUND_TARGET_SCREENSHOT_TIMEOUT_MS = 2_000; const AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS = 3_000; -const automationExecutionBudget = (timeoutMs: number): number => - timeoutMs > AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS * 2 - ? timeoutMs - AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS - : timeoutMs; +const AUTOMATION_SCREENSHOT_SETTLEMENT_GRACE_MS = 25; +export const automationExecutionBudget = (timeoutMs: number): number => + Math.min( + timeoutMs, + Math.max( + AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS * 2, + timeoutMs - AUTOMATION_TIMEOUT_RESPONSE_GRACE_MS, + ), + ); const RECORDING_FRAME_INTERVAL_MS = Math.ceil(1_000 / 12); const RECORDING_JPEG_QUALITY = 80; const PICTURE_IN_PICTURE_INITIAL_WIDTH = 480; @@ -972,9 +977,14 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, wc: Electron.WebContents, action: string, - use: (send: SendCommand, sendCleanup: SendCommand) => Effect.Effect, + use: ( + send: SendCommand, + sendCleanup: SendCommand, + operationDeadline: number, + ) => Effect.Effect, timeoutMs = DEFAULT_AUTOMATION_TIMEOUT_MS, ) { + const executionBudgetMs = automationExecutionBudget(timeoutMs); const sequence = yield* nextCounter(actionSequenceRef); const startedAt = yield* currentIso; const millis = yield* currentMillis; @@ -986,7 +996,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }; yield* pushAction(tabId, actionEvent); const epoch = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0; - const execute = Effect.fn("PreviewManager.executeControlAction")(function* () { + const execute = Effect.fn("PreviewManager.executeControlAction")(function* ( + operationDeadline: number, + ) { yield* update(tabId, { controller: "agent" }); const send: SendCommand = Effect.fn("PreviewManager.sendCommand")( function* (method, commandParams) { @@ -1034,7 +1046,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function features: [{ name: "prefers-color-scheme", value: colorScheme }], }); } - return yield* use(send, sendCleanup); + return yield* use(send, sendCleanup, operationDeadline); }); let detachOnTimeout = true; let permitAcquired = false; @@ -1073,6 +1085,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } }); const boundedExecution = Effect.gen(function* () { + const operationDeadline = (yield* currentMillis) + executionBudgetMs; // Session initialization itself sends CDP commands. Keep it inside the // operation deadline so an offscreen or suspended guest cannot retain // the synchronized session lock indefinitely and poison later actions. @@ -1082,10 +1095,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function Effect.sync(() => { detachOnTimeout = true; permitAcquired = true; - }).pipe(Effect.andThen(execute())), + }).pipe(Effect.andThen(execute(operationDeadline))), ); }).pipe( - Effect.timeoutOption(automationExecutionBudget(timeoutMs)), + Effect.timeoutOption(executionBudgetMs), Effect.flatMap((result) => Option.isNone(result) ? Effect.fail(new PreviewAutomationTimeoutError({ tabId, timeoutMs })) @@ -2848,7 +2861,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); const captureAutomationSnapshot = Effect.fn("PreviewManager.captureAutomationSnapshot")( - function* (tabId: string, wc: Electron.WebContents, send: SendCommand, background: boolean) { + function* ( + tabId: string, + wc: Electron.WebContents, + send: SendCommand, + background: boolean, + operationDeadline: number, + ) { yield* Effect.all([send("Runtime.enable"), send("Accessibility.enable")], { concurrency: 2, discard: true, @@ -2923,11 +2942,24 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const primaryScreenshotTimeoutMs = background ? AUTOMATION_BACKGROUND_TARGET_SCREENSHOT_TIMEOUT_MS : AUTOMATION_SCREENSHOT_TIMEOUT_MS; - const primaryScreenshotResult = yield* ( - background - ? captureAutomationTargetScreenshot(tabId, wc, send) - : captureAutomationScreenshot(tabId, wc, send) - ).pipe(Effect.timeoutOption(primaryScreenshotTimeoutMs), Effect.exit); + const remainingBeforePrimary = + operationDeadline - (yield* currentMillis) - AUTOMATION_SCREENSHOT_SETTLEMENT_GRACE_MS; + const fallbackReservationMs = Math.min( + AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS, + Math.max(0, Math.floor(remainingBeforePrimary / 2)), + ); + const boundedPrimaryScreenshotTimeoutMs = Math.min( + primaryScreenshotTimeoutMs, + Math.max(0, remainingBeforePrimary - fallbackReservationMs), + ); + const primaryScreenshotResult = + boundedPrimaryScreenshotTimeoutMs > 0 + ? yield* ( + background + ? captureAutomationTargetScreenshot(tabId, wc, send) + : captureAutomationScreenshot(tabId, wc, send) + ).pipe(Effect.timeoutOption(boundedPrimaryScreenshotTimeoutMs), Effect.exit) + : Exit.succeed(Option.none>()); let screenshot: PreviewAutomationSnapshot["screenshot"] = Exit.isSuccess(primaryScreenshotResult) && Option.isSome(primaryScreenshotResult.value) ? primaryScreenshotResult.value.value @@ -2939,12 +2971,22 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ? primaryScreenshotResult.cause : new PreviewAutomationTimeoutError({ tabId, - timeoutMs: primaryScreenshotTimeoutMs, + timeoutMs: boundedPrimaryScreenshotTimeoutMs, }); - const backgroundScreenshotResult = yield* captureBackgroundPage(tabId, wc, background).pipe( - Effect.timeoutOption(AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS), - Effect.exit, + const boundedFallbackScreenshotTimeoutMs = Math.min( + AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS, + Math.max( + 0, + operationDeadline - (yield* currentMillis) - AUTOMATION_SCREENSHOT_SETTLEMENT_GRACE_MS, + ), ); + const backgroundScreenshotResult = + boundedFallbackScreenshotTimeoutMs > 0 + ? yield* captureBackgroundPage(tabId, wc, background).pipe( + Effect.timeoutOption(boundedFallbackScreenshotTimeoutMs), + Effect.exit, + ) + : Exit.succeed(Option.none>()); screenshot = Exit.isSuccess(backgroundScreenshotResult) && Option.isSome(backgroundScreenshotResult.value) @@ -2955,7 +2997,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ? backgroundScreenshotResult.cause : new PreviewAutomationTimeoutError({ tabId, - timeoutMs: AUTOMATION_BACKGROUND_CAPTURE_PAGE_TIMEOUT_MS, + timeoutMs: boundedFallbackScreenshotTimeoutMs, }); yield* Effect.logWarning("Preview automation screenshot capture was unavailable.", { tabId, @@ -2990,7 +3032,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId, wc, "snapshot", - (send) => captureAutomationSnapshot(tabId, wc, send, false), + (send, _sendCleanup, operationDeadline) => + captureAutomationSnapshot(tabId, wc, send, false, operationDeadline), timeoutMs, ); if (result.detachAfterCapture) yield* detachControlSession(wc.id); @@ -3006,7 +3049,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId, wc, "snapshot", - (send) => captureAutomationSnapshot(tabId, wc, send, true), + (send, _sendCleanup, operationDeadline) => + captureAutomationSnapshot(tabId, wc, send, true, operationDeadline), timeoutMs, ); if (result.detachAfterCapture) yield* detachControlSession(wc.id); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index afcffc4bc4f..85caaae5eae 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -235,6 +235,35 @@ describe("preview automation presentation", () => { } }); + it("clamps open visibility polling to the remaining operation budget", async () => { + vi.useFakeTimers(); + const runtimeTabId = addRuntimeTab("tab-open"); + vi.stubGlobal("window", { + setTimeout, + }); + try { + const visibility = waitForBrowserSurfaceVisibility({ + threadRef, + requestId: "request-open", + tabId: "tab-open", + runtimeTabId, + timeoutMs: 40, + }); + const rejection = expect(visibility).rejects.toMatchObject({ + _tag: "PreviewAutomationVisibilityTimeoutError", + requestId: "request-open", + tabId: "tab-open", + timeoutMs: 40, + }); + + await vi.advanceTimersByTimeAsync(40); + await rejection; + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); + it("rejects an open visibility wait when the runtime guest is replaced", async () => { vi.useFakeTimers(); vi.stubGlobal("window", { diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index bbc29fc8532..09261b4cbdf 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -170,7 +170,9 @@ export async function waitForBrowserSurfaceVisibility( // Reassert the explicit show request only while that request is pending. revealPreviewAutomationTab(input.threadRef, input.tabId); } - await new Promise((resolve) => window.setTimeout(resolve, 50)); + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) break; + await new Promise((resolve) => window.setTimeout(resolve, Math.min(50, remainingMs))); } throw new PreviewAutomationVisibilityTimeoutError({ requestId: input.requestId, diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts index 23451d4a622..517f69f92d3 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts @@ -61,10 +61,23 @@ describe("previewAutomationRequestConsumer", () => { it("preserves the full execution budget for short requested timeouts", () => { expect(previewAutomationExecutionBudget(100, 250)).toBe(100); expect(previewAutomationExecutionBudget(500, 250)).toBe(500); + expect(previewAutomationExecutionBudget(501, 250)).toBe(500); + expect(previewAutomationExecutionBudget(750, 250)).toBe(500); + expect(previewAutomationExecutionBudget(751, 250)).toBe(501); expect(previewAutomationExecutionBudget(1_000, 250)).toBe(750); expect(previewAutomationExecutionBudget(1_000)).toBe(750); }); + it("keeps execution budgets monotonic as requested timeouts increase", () => { + const budgets = Array.from({ length: 1_001 }, (_, timeoutMs) => + previewAutomationExecutionBudget(timeoutMs, 250), + ); + + expect(budgets.every((budget, index) => index === 0 || budget >= budgets[index - 1]!)).toBe( + true, + ); + }); + it("reports an expired operation budget instead of clamping it to one millisecond", () => { expect(previewAutomationRemainingBudget(1_000, 15_000, 999)).toBe(1); expect(previewAutomationRemainingBudget(1_000, 15_000, 1_001)).toBe(-1); diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts index 77fb6aac8c7..b80a8f98a05 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts @@ -20,7 +20,7 @@ export const PREVIEW_AUTOMATION_RESPONSE_GRACE_MS = 250; export const previewAutomationExecutionBudget = ( timeoutMs: number, responseGraceMs = PREVIEW_AUTOMATION_RESPONSE_GRACE_MS, -): number => (timeoutMs > responseGraceMs * 2 ? timeoutMs - responseGraceMs : timeoutMs); +): number => Math.min(timeoutMs, Math.max(responseGraceMs * 2, timeoutMs - responseGraceMs)); export const previewAutomationRemainingBudget = ( operationDeadline: number, From b40f6f9b1b63dfd54aba314753fe5c01f738f5c9 Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Tue, 28 Jul 2026 18:01:57 +0100 Subject: [PATCH 28/43] Preserve sessions when screenshot capture is skipped - Distinguish exhausted-budget skips from actual CDP timeouts - Verify the healthy debugger session is reused afterward --- BRANCH_DETAILS.md | 2 +- apps/desktop/src/preview/Manager.test.ts | 27 ++++++++++++++++++++++++ apps/desktop/src/preview/Manager.ts | 20 ++++++++++-------- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 27ad1a0c15f..cff237ea95b 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -19,7 +19,7 @@ Expected behavior: - Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout without making the execution budget shrink when the caller increases a short timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session when initialization or an acquired permit may have left a CDP command pending. A request that times out while queued behind another action does not detach that action's shared debugger session. - Click, type, and wait operations propagate their caller-supplied timeout into the desktop control-session boundary. Operations without a caller timeout retain the bounded desktop default. -- Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Primary and fallback screenshot waits are clamped to the remaining control-session deadline, with budget reserved for fallback and result settlement, so a tight caller deadline can still return semantic data instead of being preempted by the outer session timeout. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, timed-out CDP capture resets the session after releasing its control permit, and the semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. +- Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Primary and fallback screenshot waits are clamped to the remaining control-session deadline, with budget reserved for fallback and result settlement, so a tight caller deadline can still return semantic data instead of being preempted by the outer session timeout. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, an actually timed-out CDP capture resets the session after releasing its control permit, and a capture skipped before CDP runs leaves the healthy session attached. The semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. A persisted non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. - Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the server epoch replaces the runtime guest while staging is pending, the snapshot fails immediately with `PreviewAutomationTargetUnavailableError` instead of waiting on the stale staging marker. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 5810051d339..8dbceaebdbe 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -315,6 +315,7 @@ describe("PreviewManager", () => { createFromBuffer.mockReturnValue(image); let attached = false; let captureMode: "available" | "failure" | "timeout" = "available"; + let accessibilityGate: Promise | null = null; const attach = vi.fn(() => { attached = true; }); @@ -337,6 +338,7 @@ describe("PreviewManager", () => { }; } if (method === "Accessibility.getFullAXTree") { + if (accessibilityGate !== null) await accessibilityGate; return { nodes: [] }; } if (method === "Target.getTargetInfo") { @@ -501,6 +503,31 @@ describe("PreviewManager", () => { }); expect(capturePage).toHaveBeenCalledOnce(); expect(detach).toHaveBeenCalledTimes(2); + + capturePage.mockClear(); + captureMode = "available"; + let releaseAccessibility!: () => void; + accessibilityGate = new Promise((resolve) => { + releaseAccessibility = resolve; + }); + const skippedCapture = yield* manager + .automationSnapshot("tab_snapshot", false, 1_000) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + yield* TestClock.adjust(730); + releaseAccessibility(); + accessibilityGate = null; + + expect((yield* Fiber.join(skippedCapture)).screenshot).toBeNull(); + expect(capturePage).not.toHaveBeenCalled(); + expect(detach).toHaveBeenCalledTimes(2); + expect(attach).toHaveBeenCalledTimes(3); + + expect((yield* manager.automationSnapshot("tab_snapshot")).screenshot).toMatchObject({ + width: 640, + height: 360, + }); + expect(attach).toHaveBeenCalledTimes(3); }), ), ); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index f3834126af7..c1f4b796220 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2952,20 +2952,22 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function primaryScreenshotTimeoutMs, Math.max(0, remainingBeforePrimary - fallbackReservationMs), ); - const primaryScreenshotResult = - boundedPrimaryScreenshotTimeoutMs > 0 - ? yield* ( - background - ? captureAutomationTargetScreenshot(tabId, wc, send) - : captureAutomationScreenshot(tabId, wc, send) - ).pipe(Effect.timeoutOption(boundedPrimaryScreenshotTimeoutMs), Effect.exit) - : Exit.succeed(Option.none>()); + const primaryScreenshotAttempted = boundedPrimaryScreenshotTimeoutMs > 0; + const primaryScreenshotResult = primaryScreenshotAttempted + ? yield* ( + background + ? captureAutomationTargetScreenshot(tabId, wc, send) + : captureAutomationScreenshot(tabId, wc, send) + ).pipe(Effect.timeoutOption(boundedPrimaryScreenshotTimeoutMs), Effect.exit) + : Exit.succeed(Option.none>()); let screenshot: PreviewAutomationSnapshot["screenshot"] = Exit.isSuccess(primaryScreenshotResult) && Option.isSome(primaryScreenshotResult.value) ? primaryScreenshotResult.value.value : null; const detachAfterCapture = - Exit.isSuccess(primaryScreenshotResult) && Option.isNone(primaryScreenshotResult.value); + primaryScreenshotAttempted && + Exit.isSuccess(primaryScreenshotResult) && + Option.isNone(primaryScreenshotResult.value); if (screenshot === null) { const primaryFailure = Exit.isFailure(primaryScreenshotResult) ? primaryScreenshotResult.cause From 864d9a238e5dad4b15c1eef69c098325a7dc9815 Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Wed, 29 Jul 2026 11:44:42 +0100 Subject: [PATCH 29/43] Serialize pairing and scope browser surface renders - Keep retained webviews subscribed to their own runtime tab state - Queue pairing exchanges and retain pending state across the queue - Add focused regressions and integrated verification notes --- BRANCH_DETAILS.md | 6 ++- apps/web/src/browser/HostedBrowserWebview.tsx | 5 +-- .../src/browser/browserSurfaceStore.test.ts | 44 ++++++++++-------- apps/web/src/browser/browserSurfaceStore.ts | 6 +-- .../auth/PairingRouteSurface.logic.test.ts | 42 ++++++++++++++++- .../auth/PairingRouteSurface.logic.ts | 19 ++++++++ .../components/auth/PairingRouteSurface.tsx | 45 ++++++++++++------- 7 files changed, 121 insertions(+), 46 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index cff237ea95b..048f0112471 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -27,7 +27,8 @@ Expected behavior: - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget, and the transition into grace reservation remains monotonic as requested timeouts increase. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and both visibility polling and stable-presentation dwell contract to fit short deadlines instead of overshooting them or requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. - A newly created preview tab applies its server snapshot and assigned tab id, applies the upstream 1280×800 automation viewport when the server snapshot still uses `fill`, initiates any requested selection, and acknowledges server-side creation without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across same-server route hydration or session reconciliation instead of accepting one transient visible frame. A server-epoch change denotes a replacement runtime guest and aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the new guest. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for `open`. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. -- The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. +- Retained browser guests subscribe only to render state for their own epoch-scoped runtime tab. Presentation or background-capture updates for another tab do not rerender every mounted `HostedBrowserWebview`; background staging derives its viewport-fitted rectangle from the selected tab's own stable rectangle. +- The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. Multiple tokens received while an exchange is pending are serialized, and the submitting state remains active until every queued exchange settles. - `dev:desktop` derives `T3CODE_DESKTOP_USER_DATA_DIR=/userdata/electron` whenever the runner has an explicit base directory. Desktop configuration resolves that override to an absolute path, and app identity uses it before legacy migration or the normal Electron user-data default. This keeps an isolated worktree dev desktop from reusing an installed or earlier development profile whose incompatible IndexedDB schema can prevent the renderer from starting. Packaged/default startup remains unchanged when no override is supplied. Current limitations: @@ -68,11 +69,12 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.t Current verification: -- The focused command above passed all 223 tests. +- The focused command above passed all 225 tests, including current-tab-only browser-surface selection and serialized pairing submissions. - The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 42 tests with explicit runtime guest identity, including immediate open, monotonic short-deadline handling, deadline-clamped visibility polling, and snapshot rejection after a server-epoch replacement. - Desktop, server, contracts, and scripts typechecks completed without type errors. The web typecheck currently reaches only three `RegistryContext` typing errors that are also present on `upstream/main`. - An isolated worktree `dev:desktop` using the user-data override paired successfully and stayed usable through repeated renderer/CDP inspection and raster capture. An archive worktree client loaded its seeded project against that host without the prior IndexedDB `VersionError` or host-interface disappearance. - An isolated web client on ports `5744`/`13784` completed first-navigation pairing, loaded the seeded Preview Reliability thread, and rendered the right-panel surface chooser. Its non-Electron Browser surface was unavailable as described under current limitations. +- A second isolated web pass on ports `5744`/`13784` delayed the first browser-session exchange, injected another fragment token while it was pending, and verified from in-page request timestamps that the second exchange started only after the first finished; the client then loaded the authenticated app. ## Development Ports diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index 04913875da3..22de1c31804 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -10,7 +10,6 @@ import { cn } from "~/lib/utils"; import { resolveBrowserSurfaceBackgroundCaptureRect, - resolveBrowserSurfacePanelRect, selectBrowserSurfaceRenderState, useBrowserSurfaceStore, } from "./browserSurfaceStore"; @@ -72,11 +71,11 @@ export function HostedBrowserWebview(props: { fitSourceContent: surface.fitSourceContent, fittedSourceContent: surface.fittedSourceContent, rect: surface.backgroundCapture - ? resolveBrowserSurfaceBackgroundCaptureRect(surface.byTabId, runtimeTabId, { + ? resolveBrowserSurfaceBackgroundCaptureRect(surface.rect, { width: window.innerWidth, height: window.innerHeight, }) - : resolveBrowserSurfacePanelRect(surface.byTabId, runtimeTabId), + : surface.rect, visible: surface.visible, }; usePreviewBridge({ threadRef, tabId, runtimeTabId }); diff --git a/apps/web/src/browser/browserSurfaceStore.test.ts b/apps/web/src/browser/browserSurfaceStore.test.ts index 33d7da7bd41..e58e11aeb51 100644 --- a/apps/web/src/browser/browserSurfaceStore.test.ts +++ b/apps/web/src/browser/browserSurfaceStore.test.ts @@ -35,23 +35,41 @@ describe("browserSurfaceStore", () => { ).toBeUndefined(); }); - it("selects stable inputs while a background capture rect is derived", () => { + it("selects only the current tab's stable render inputs", () => { const release = acquireBrowserSurfaceBackgroundCapture("tab-background"); const state = useBrowserSurfaceStore.getState(); const first = selectBrowserSurfaceRenderState(state, "tab-background"); - const second = selectBrowserSurfaceRenderState(state, "tab-background"); + useBrowserSurfaceStore.setState({ + byTabId: { + ...state.byTabId, + unrelated: { + rect: { x: 10, y: 20, width: 300, height: 200 }, + visible: true, + content: null, + fittedSourceContent: null, + fitSourceContent: false, + cornerRadius: 0, + updatedAt: 1, + owner: null, + }, + }, + }); + const second = selectBrowserSurfaceRenderState( + useBrowserSurfaceStore.getState(), + "tab-background", + ); expect(shallow(first, second)).toBe(true); expect(first).toEqual({ - byTabId: state.byTabId, backgroundCapture: true, content: null, cornerRadius: 0, fitSourceContent: false, fittedSourceContent: null, + rect: null, visible: false, }); - expect(first).not.toHaveProperty("rect"); + expect(first).not.toHaveProperty("byTabId"); release(); }); @@ -164,7 +182,7 @@ describe("browserSurfaceStore", () => { it("stages a never-presented background tab inside the renderer viewport", () => { expect( - resolveBrowserSurfaceBackgroundCaptureRect({}, "never-presented", { + resolveBrowserSurfaceBackgroundCaptureRect(null, { width: 1440, height: 900, }), @@ -178,7 +196,7 @@ describe("browserSurfaceStore", () => { it("fits background capture staging to a smaller renderer viewport", () => { expect( - resolveBrowserSurfaceBackgroundCaptureRect({}, "never-presented", { + resolveBrowserSurfaceBackgroundCaptureRect(null, { width: 800, height: 600, }), @@ -193,19 +211,7 @@ describe("browserSurfaceStore", () => { it("clamps a stale presented rectangle to the current renderer viewport", () => { expect( resolveBrowserSurfaceBackgroundCaptureRect( - { - active: { - rect: { x: 1_000, y: 700, width: 900, height: 640 }, - visible: true, - content: null, - fittedSourceContent: null, - fitSourceContent: false, - cornerRadius: 0, - updatedAt: 1, - owner: null, - }, - }, - "hidden", + { x: 1_000, y: 700, width: 900, height: 640 }, { width: 800, height: 600 }, ), ).toEqual({ diff --git a/apps/web/src/browser/browserSurfaceStore.ts b/apps/web/src/browser/browserSurfaceStore.ts index 5cda4c74a46..d85588c76fe 100644 --- a/apps/web/src/browser/browserSurfaceStore.ts +++ b/apps/web/src/browser/browserSurfaceStore.ts @@ -59,12 +59,12 @@ export function selectBrowserSurfaceRenderState( ) { const current = state.byTabId[tabId]; return { - byTabId: state.byTabId, backgroundCapture: (state.backgroundCaptureCountByTabId[tabId] ?? 0) > 0, content: current?.content ?? null, cornerRadius: current?.cornerRadius ?? 0, fitSourceContent: current?.fitSourceContent ?? false, fittedSourceContent: current?.fittedSourceContent ?? null, + rect: current?.rect ?? null, visible: current?.visible ?? false, }; } @@ -78,15 +78,13 @@ export function resolveBrowserSurfacePanelRect( } export function resolveBrowserSurfaceBackgroundCaptureRect( - byTabId: Readonly>, - tabId: string, + presentedRect: BrowserSurfaceRect | null, viewport: { readonly width: number; readonly height: number }, ): BrowserSurfaceRect { const viewportWidth = Number.isFinite(viewport.width) && viewport.width > 0 ? Math.round(viewport.width) : 1280; const viewportHeight = Number.isFinite(viewport.height) && viewport.height > 0 ? Math.round(viewport.height) : 800; - const presentedRect = resolveBrowserSurfacePanelRect(byTabId, tabId); if (presentedRect) { const width = Math.max(1, Math.min(Math.round(presentedRect.width), viewportWidth)); const height = Math.max(1, Math.min(Math.round(presentedRect.height), viewportHeight)); diff --git a/apps/web/src/components/auth/PairingRouteSurface.logic.test.ts b/apps/web/src/components/auth/PairingRouteSurface.logic.test.ts index 4c8657aff62..ea08441f322 100644 --- a/apps/web/src/components/auth/PairingRouteSurface.logic.test.ts +++ b/apps/web/src/components/auth/PairingRouteSurface.logic.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vite-plus/test"; -import { claimPairingToken } from "./PairingRouteSurface.logic"; +import { claimPairingToken, createPairingSubmissionQueue } from "./PairingRouteSurface.logic"; describe("claimPairingToken", () => { it("claims each pairing token once", () => { @@ -14,4 +14,44 @@ describe("claimPairingToken", () => { it("ignores a URL without a pairing token", () => { expect(claimPairingToken(null, new Set())).toBeNull(); }); + + it("serializes pairing submissions", async () => { + const queue = createPairingSubmissionQueue(); + const events: Array = []; + let finishFirst: (() => void) | undefined; + + const first = queue.run( + () => + new Promise((resolve) => { + events.push("first:start"); + finishFirst = () => { + events.push("first:finish"); + resolve(); + }; + }), + ); + const second = queue.run(async () => { + events.push("second:start"); + }); + + await Promise.resolve(); + expect(events).toEqual(["first:start"]); + + finishFirst?.(); + await first; + await second; + expect(events).toEqual(["first:start", "first:finish", "second:start"]); + }); + + it("continues the queue after a rejected pairing submission", async () => { + const queue = createPairingSubmissionQueue(); + const expectedError = new Error("pairing failed"); + const first = queue.run(async () => { + throw expectedError; + }); + const second = queue.run(async () => "paired"); + + await expect(first).rejects.toBe(expectedError); + await expect(second).resolves.toBe("paired"); + }); }); diff --git a/apps/web/src/components/auth/PairingRouteSurface.logic.ts b/apps/web/src/components/auth/PairingRouteSurface.logic.ts index 6b1e729a592..04c4c3e33ab 100644 --- a/apps/web/src/components/auth/PairingRouteSurface.logic.ts +++ b/apps/web/src/components/auth/PairingRouteSurface.logic.ts @@ -6,3 +6,22 @@ export function claimPairingToken( attemptedTokens.add(token); return token; } + +export interface PairingSubmissionQueue { + readonly run: (submit: () => Promise) => Promise; +} + +export function createPairingSubmissionQueue(): PairingSubmissionQueue { + let tail: Promise = Promise.resolve(); + + return { + run: (submit: () => Promise) => { + const result = tail.then(submit, submit); + tail = result.then( + () => undefined, + () => undefined, + ); + return result; + }, + }; +} diff --git a/apps/web/src/components/auth/PairingRouteSurface.tsx b/apps/web/src/components/auth/PairingRouteSurface.tsx index b066fdede47..49d99f25501 100644 --- a/apps/web/src/components/auth/PairingRouteSurface.tsx +++ b/apps/web/src/components/auth/PairingRouteSurface.tsx @@ -13,7 +13,7 @@ import { readHostedPairingRequest } from "../../hostedPairing"; import { Button } from "../ui/button"; import { Input } from "../ui/input"; import { useAtomCommand } from "../../state/use-atom-command"; -import { claimPairingToken } from "./PairingRouteSurface.logic"; +import { claimPairingToken, createPairingSubmissionQueue } from "./PairingRouteSurface.logic"; export function PairingPendingSurface() { return ( @@ -51,30 +51,41 @@ export function PairingRouteSurface({ const [credential, setCredential] = useState(() => peekPairingTokenFromUrl() ?? ""); const [errorMessage, setErrorMessage] = useState(initialErrorMessage ?? ""); const [isSubmitting, setIsSubmitting] = useState(false); + const [submissionQueue] = useState(createPairingSubmissionQueue); const attemptedPairingTokensRef = useRef(new Set()); + const pendingSubmissionCountRef = useRef(0); const submitCredential = useCallback( async (nextCredential: string) => { + pendingSubmissionCountRef.current += 1; setIsSubmitting(true); - setErrorMessage(""); - const submitError = await submitServerAuthCredential(nextCredential).then( - () => null, - (error) => errorMessageFromUnknown(error), - ); - - setIsSubmitting(false); - - if (submitError) { - setErrorMessage(submitError); - return; + try { + await submissionQueue.run(async () => { + setErrorMessage(""); + + const submitError = await submitServerAuthCredential(nextCredential).then( + () => null, + (error) => errorMessageFromUnknown(error), + ); + + if (submitError) { + setErrorMessage(submitError); + return; + } + + startTransition(() => { + onAuthenticated(); + }); + }); + } finally { + pendingSubmissionCountRef.current -= 1; + if (pendingSubmissionCountRef.current === 0) { + setIsSubmitting(false); + } } - - startTransition(() => { - onAuthenticated(); - }); }, - [onAuthenticated], + [onAuthenticated, submissionQueue], ); const handleSubmit = useCallback( From 35aa79803a9677d3127d7926f38ad90d435c63b7 Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Wed, 29 Jul 2026 12:01:31 +0100 Subject: [PATCH 30/43] Serialize control session timeout recovery - Detach poisoned debugger sessions before releasing control permits - Retry queued operations when their acquired session was retired - Cover screenshot timeout recovery with queued evaluation ordering --- BRANCH_DETAILS.md | 5 +- apps/desktop/src/preview/Manager.test.ts | 13 +++++ apps/desktop/src/preview/Manager.ts | 72 ++++++++++++++---------- 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 048f0112471..2965443e74b 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -17,9 +17,9 @@ This branch does not add a second recording/PiP capture lifecycle or another hid Expected behavior: -- Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout without making the execution budget shrink when the caller increases a short timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session when initialization or an acquired permit may have left a CDP command pending. A request that times out while queued behind another action does not detach that action's shared debugger session. +- Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout without making the execution budget shrink when the caller increases a short timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session while still holding an acquired control permit when a CDP command may be pending. Session removal and debugger teardown are atomic with respect to new session acquisition, and operations already queued on the retired semaphore detect that stale session and retry against its replacement. A request that times out while queued behind another action does not detach that action's shared debugger session. - Click, type, and wait operations propagate their caller-supplied timeout into the desktop control-session boundary. Operations without a caller timeout retain the bounded desktop default. -- Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Primary and fallback screenshot waits are clamped to the remaining control-session deadline, with budget reserved for fallback and result settlement, so a tight caller deadline can still return semantic data instead of being preempted by the outer session timeout. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, an actually timed-out CDP capture resets the session after releasing its control permit, and a capture skipped before CDP runs leaves the healthy session attached. The semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. +- Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Primary and fallback screenshot waits are clamped to the remaining control-session deadline, with budget reserved for fallback and result settlement, so a tight caller deadline can still return semantic data instead of being preempted by the outer session timeout. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, an actually timed-out CDP capture resets the session before releasing its control permit, queued work reattaches before issuing its first command, and a capture skipped before CDP runs leaves the healthy session attached. The semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. A persisted non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. - Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the server epoch replaces the runtime guest while staging is pending, the snapshot fails immediately with `PreviewAutomationTargetUnavailableError` instead of waiting on the stale staging marker. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. @@ -70,6 +70,7 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.t Current verification: - The focused command above passed all 225 tests, including current-tab-only browser-surface selection and serialized pairing submissions. +- The desktop manager's 35 focused tests additionally verify that a screenshot timeout detaches the poisoned debugger session before a queued evaluation reattaches and sends its first command. - The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 42 tests with explicit runtime guest identity, including immediate open, monotonic short-deadline handling, deadline-clamped visibility polling, and snapshot rejection after a server-epoch replacement. - Desktop, server, contracts, and scripts typechecks completed without type errors. The web typecheck currently reaches only three `RegistryContext` typing errors that are also present on `upstream/main`. - An isolated worktree `dev:desktop` using the user-data override paired successfully and stayed usable through repeated renderer/CDP inspection and raster capture. An archive worktree client loaded its seeded project against that host without the prior IndexedDB `VersionError` or host-interface disappearance. diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 8dbceaebdbe..293bb6ac8eb 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -316,15 +316,19 @@ describe("PreviewManager", () => { let attached = false; let captureMode: "available" | "failure" | "timeout" = "available"; let accessibilityGate: Promise | null = null; + const controlEvents: Array = []; const attach = vi.fn(() => { attached = true; + controlEvents.push("attach"); }); const detach = vi.fn(() => { attached = false; + controlEvents.push("detach"); }); const capturePage = vi.fn(async () => image); const sendCommand = vi.fn(async (method: string): Promise => { if (method === "Runtime.evaluate") { + controlEvents.push("evaluate"); return { result: { value: { @@ -475,13 +479,22 @@ describe("PreviewManager", () => { const timedOutCapture = yield* manager .automationSnapshot("tab_snapshot") .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + const queuedAfterTimedOutCapture = yield* manager + .automationEvaluate("tab_snapshot", { expression: "document.title" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; yield* TestClock.adjust(5_000); expect((yield* Fiber.join(timedOutCapture)).screenshot).toMatchObject({ width: 640, height: 360, }); + expect(yield* Fiber.join(queuedAfterTimedOutCapture)).toMatchObject({ + title: "Example", + }); expect(capturePage).toHaveBeenCalledOnce(); expect(detach).toHaveBeenCalledOnce(); + expect(controlEvents.slice(-3)).toEqual(["detach", "attach", "evaluate"]); captureMode = "available"; const recovered = yield* manager.automationSnapshot("tab_snapshot"); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index c1f4b796220..f7e046e52e3 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -391,6 +391,8 @@ interface BrowserControlSession { ) => void; } +const STALE_BROWSER_CONTROL_SESSION = Symbol("StaleBrowserControlSession"); + interface BrowserDiagnostics { readonly consoleEntries: ReadonlyArray; readonly networkEntries: ReadonlyArray; @@ -819,16 +821,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const detachControlSession = Effect.fn("PreviewManager.detachControlSession")(function* ( webContentsId: number, ) { - const control = yield* SynchronizedRef.modify(controlSessionsRef, (sessions) => [ - sessions.get(webContentsId), - replaceMap(sessions, (copy) => { - copy.delete(webContentsId); - }), - ]); - if (control) { - yield* Scope.close(control.scope, Exit.void).pipe(Effect.ignore); - return; - } + const detached = yield* SynchronizedRef.modifyEffect(controlSessionsRef, (sessions) => { + const control = sessions.get(webContentsId); + if (!control) return Effect.succeed([false, sessions] as const); + return Scope.close(control.scope, Exit.void).pipe( + Effect.ignore, + Effect.as([ + true, + replaceMap(sessions, (copy) => { + copy.delete(webContentsId); + }), + ] as const), + ); + }); + if (detached) return; yield* Ref.update(diagnosticsRef, (diagnostics) => replaceMap(diagnostics, (copy) => { copy.delete(webContentsId); @@ -1086,17 +1092,29 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); const boundedExecution = Effect.gen(function* () { const operationDeadline = (yield* currentMillis) + executionBudgetMs; - // Session initialization itself sends CDP commands. Keep it inside the - // operation deadline so an offscreen or suspended guest cannot retain - // the synchronized session lock indefinitely and poison later actions. - const control = yield* ensureControlSession(wc); - detachOnTimeout = false; - return yield* control.semaphore.withPermit( - Effect.sync(() => { - detachOnTimeout = true; - permitAcquired = true; - }).pipe(Effect.andThen(execute(operationDeadline))), - ); + while (true) { + // Session initialization itself sends CDP commands. Keep it inside the + // operation deadline so an offscreen or suspended guest cannot retain + // the synchronized session lock indefinitely and poison later actions. + const control = yield* ensureControlSession(wc); + detachOnTimeout = false; + const result = yield* control.semaphore.withPermit( + Effect.gen(function* () { + const currentControl = (yield* SynchronizedRef.get(controlSessionsRef)).get(wc.id); + if (currentControl !== control) return STALE_BROWSER_CONTROL_SESSION; + detachOnTimeout = true; + permitAcquired = true; + return yield* execute(operationDeadline); + }).pipe( + Effect.onInterrupt(() => + Effect.sync(() => { + detachOnTimeout = false; + }).pipe(Effect.andThen(detachControlSession(wc.id))), + ), + ), + ); + if (result !== STALE_BROWSER_CONTROL_SESSION) return result; + } }).pipe( Effect.timeoutOption(executionBudgetMs), Effect.flatMap((result) => @@ -3011,6 +3029,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } } const browserDiagnostics = diagnostics.get(wc.id); + if (detachAfterCapture) yield* detachControlSession(wc.id); return { ...page, accessibilityTree: accessibility, @@ -3018,7 +3037,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function networkEntries: [...(browserDiagnostics?.networkEntries ?? [])], actionTimeline: [...(timelines.get(tabId) ?? [])], screenshot, - detachAfterCapture, }; }, ); @@ -3030,7 +3048,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ) { const wc = yield* requireWebContents(tabId); if (!background) { - const result = yield* withControlSession( + return yield* withControlSession( tabId, wc, "snapshot", @@ -3038,16 +3056,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function captureAutomationSnapshot(tabId, wc, send, false, operationDeadline), timeoutMs, ); - if (result.detachAfterCapture) yield* detachControlSession(wc.id); - const { detachAfterCapture: _, ...snapshot } = result; - return snapshot; } // The renderer briefly stages a non-selected guest so Chromium can expose // its composited pixels. Do not focus the guest or send Page.bringToFront: // Electron can otherwise promote the native guest surface above the host // UI while ignoring the staging wrapper's opacity. - const result = yield* withControlSession( + return yield* withControlSession( tabId, wc, "snapshot", @@ -3055,9 +3070,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function captureAutomationSnapshot(tabId, wc, send, true, operationDeadline), timeoutMs, ); - if (result.detachAfterCapture) yield* detachControlSession(wc.id); - const { detachAfterCapture: _, ...snapshot } = result; - return snapshot; }); const resolveClickPoint = Effect.fn("PreviewManager.resolveClickPoint")(function* ( From 7a576c2850b7594c815d62e2227c06b1e02f0fc1 Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Wed, 29 Jul 2026 12:16:19 +0100 Subject: [PATCH 31/43] Bind timeout cleanup to control sessions - Reset only the exact session held by an interrupted operation - Keep stale waiters from installing replacement-detach finalizers - Pass session-bound reset effects through snapshot capture --- BRANCH_DETAILS.md | 2 +- apps/desktop/src/preview/Manager.ts | 46 ++++++++++++----------------- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 2965443e74b..b56a0b0243f 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -17,7 +17,7 @@ This branch does not add a second recording/PiP capture lifecycle or another hid Expected behavior: -- Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout without making the execution budget shrink when the caller increases a short timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session while still holding an acquired control permit when a CDP command may be pending. Session removal and debugger teardown are atomic with respect to new session acquisition, and operations already queued on the retired semaphore detect that stale session and retry against its replacement. A request that times out while queued behind another action does not detach that action's shared debugger session. +- Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout without making the execution budget shrink when the caller increases a short timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session while still holding an acquired control permit when a CDP command may be pending. Session removal and debugger teardown are atomic with respect to new session acquisition and bound to the exact acquired session, so late interruption or snapshot cleanup cannot detach a healthy replacement. Operations already queued on the retired semaphore detect that stale session and retry against its replacement. A request that times out while queued behind another action does not detach that action's shared debugger session. - Click, type, and wait operations propagate their caller-supplied timeout into the desktop control-session boundary. Operations without a caller timeout retain the bounded desktop default. - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Primary and fallback screenshot waits are clamped to the remaining control-session deadline, with budget reserved for fallback and result settlement, so a tight caller deadline can still return semantic data instead of being preempted by the outer session timeout. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, an actually timed-out CDP capture resets the session before releasing its control permit, queued work reattaches before issuing its first command, and a capture skipped before CDP runs leaves the healthy session attached. The semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. A persisted non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index f7e046e52e3..2ff37a705ed 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -820,10 +820,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const detachControlSession = Effect.fn("PreviewManager.detachControlSession")(function* ( webContentsId: number, + expectedControl?: BrowserControlSession, ) { const detached = yield* SynchronizedRef.modifyEffect(controlSessionsRef, (sessions) => { const control = sessions.get(webContentsId); - if (!control) return Effect.succeed([false, sessions] as const); + if (!control || (expectedControl !== undefined && control !== expectedControl)) { + return Effect.succeed([false, sessions] as const); + } return Scope.close(control.scope, Exit.void).pipe( Effect.ignore, Effect.as([ @@ -834,7 +837,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ] as const), ); }); - if (detached) return; + if (detached || expectedControl !== undefined) return; yield* Ref.update(diagnosticsRef, (diagnostics) => replaceMap(diagnostics, (copy) => { copy.delete(webContentsId); @@ -987,6 +990,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function send: SendCommand, sendCleanup: SendCommand, operationDeadline: number, + resetControlSession: Effect.Effect, ) => Effect.Effect, timeoutMs = DEFAULT_AUTOMATION_TIMEOUT_MS, ) { @@ -1004,6 +1008,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const epoch = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0; const execute = Effect.fn("PreviewManager.executeControlAction")(function* ( operationDeadline: number, + control: BrowserControlSession, ) { yield* update(tabId, { controller: "agent" }); const send: SendCommand = Effect.fn("PreviewManager.sendCommand")( @@ -1052,9 +1057,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function features: [{ name: "prefers-color-scheme", value: colorScheme }], }); } - return yield* use(send, sendCleanup, operationDeadline); + return yield* use(send, sendCleanup, operationDeadline, detachControlSession(wc.id, control)); }); - let detachOnTimeout = true; let permitAcquired = false; const finalize = Effect.fn("PreviewManager.finalizeControlAction")(function* ( exit: Exit.Exit, @@ -1097,21 +1101,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function // operation deadline so an offscreen or suspended guest cannot retain // the synchronized session lock indefinitely and poison later actions. const control = yield* ensureControlSession(wc); - detachOnTimeout = false; const result = yield* control.semaphore.withPermit( Effect.gen(function* () { const currentControl = (yield* SynchronizedRef.get(controlSessionsRef)).get(wc.id); if (currentControl !== control) return STALE_BROWSER_CONTROL_SESSION; - detachOnTimeout = true; permitAcquired = true; - return yield* execute(operationDeadline); - }).pipe( - Effect.onInterrupt(() => - Effect.sync(() => { - detachOnTimeout = false; - }).pipe(Effect.andThen(detachControlSession(wc.id))), - ), - ), + return yield* execute(operationDeadline, control).pipe( + Effect.onInterrupt(() => detachControlSession(wc.id, control)), + ); + }), ); if (result !== STALE_BROWSER_CONTROL_SESSION) return result; } @@ -1123,14 +1121,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function : Effect.succeed(result.value), ), ); - return yield* boundedExecution.pipe( - Effect.onExit(finalize), - Effect.tapError((error) => - isPreviewAutomationTimeoutError(error) && detachOnTimeout - ? detachControlSession(wc.id) - : Effect.void, - ), - ); + return yield* boundedExecution.pipe(Effect.onExit(finalize)); }); const evaluateWithDebugger = ( @@ -2885,6 +2876,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function send: SendCommand, background: boolean, operationDeadline: number, + resetControlSession: Effect.Effect, ) { yield* Effect.all([send("Runtime.enable"), send("Accessibility.enable")], { concurrency: 2, @@ -3029,7 +3021,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } } const browserDiagnostics = diagnostics.get(wc.id); - if (detachAfterCapture) yield* detachControlSession(wc.id); + if (detachAfterCapture) yield* resetControlSession; return { ...page, accessibilityTree: accessibility, @@ -3052,8 +3044,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId, wc, "snapshot", - (send, _sendCleanup, operationDeadline) => - captureAutomationSnapshot(tabId, wc, send, false, operationDeadline), + (send, _sendCleanup, operationDeadline, resetControlSession) => + captureAutomationSnapshot(tabId, wc, send, false, operationDeadline, resetControlSession), timeoutMs, ); } @@ -3066,8 +3058,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId, wc, "snapshot", - (send, _sendCleanup, operationDeadline) => - captureAutomationSnapshot(tabId, wc, send, true, operationDeadline), + (send, _sendCleanup, operationDeadline, resetControlSession) => + captureAutomationSnapshot(tabId, wc, send, true, operationDeadline, resetControlSession), timeoutMs, ); }); From 1ff5aed8dd607f26f12c0e9d5d5790b606d8fb66 Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Wed, 29 Jul 2026 22:36:41 +0100 Subject: [PATCH 32/43] Keep background capture selection-aware - Treat stale visible surfaces as inactive after selection changes - Preserve staged capture markers and transparent presentation - Add focused presentation regressions and branch documentation --- BRANCH_DETAILS.md | 4 +-- apps/web/src/browser/HostedBrowserWebview.tsx | 22 +++++++++++-- .../browser/hostedBrowserWebviewStyle.test.ts | 31 +++++++++++++++++++ .../src/browser/hostedBrowserWebviewStyle.ts | 16 ++++++++++ 4 files changed, 68 insertions(+), 5 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index b56a0b0243f..7cec55fff0a 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -27,7 +27,7 @@ Expected behavior: - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget, and the transition into grace reservation remains monotonic as requested timeouts increase. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and both visibility polling and stable-presentation dwell contract to fit short deadlines instead of overshooting them or requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. - A newly created preview tab applies its server snapshot and assigned tab id, applies the upstream 1280×800 automation viewport when the server snapshot still uses `fill`, initiates any requested selection, and acknowledges server-side creation without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across same-server route hydration or session reconciliation instead of accepting one transient visible frame. A server-epoch change denotes a replacement runtime guest and aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the new guest. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for `open`. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. -- Retained browser guests subscribe only to render state for their own epoch-scoped runtime tab. Presentation or background-capture updates for another tab do not rerender every mounted `HostedBrowserWebview`; background staging derives its viewport-fitted rectangle from the selected tab's own stable rectangle. +- Retained browser guests subscribe only to render state for their own epoch-scoped runtime tab, plus their thread's active panel and mini-player selection. Presentation or background-capture updates for another tab do not rerender every mounted `HostedBrowserWebview`; selection changes override a stale surface-visible flag so background staging remains nearly transparent and exposes its readiness marker, and staging derives its viewport-fitted rectangle from the target tab's own stable rectangle. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. Multiple tokens received while an exchange is pending are serialized, and the submitting state remains active until every queued exchange settles. - `dev:desktop` derives `T3CODE_DESKTOP_USER_DATA_DIR=/userdata/electron` whenever the runner has an explicit base directory. Desktop configuration resolves that override to an absolute path, and app identity uses it before legacy migration or the normal Electron user-data default. This keeps an isolated worktree dev desktop from reusing an installed or earlier development profile whose incompatible IndexedDB schema can prevent the renderer from starting. Packaged/default startup remains unchanged when no override is supplied. @@ -69,7 +69,7 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.t Current verification: -- The focused command above passed all 225 tests, including current-tab-only browser-surface selection and serialized pairing submissions. +- The focused command above passed all 228 tests, including current-tab-only browser-surface selection, stale-visible background staging after selection changes, and serialized pairing submissions. - The desktop manager's 35 focused tests additionally verify that a screenshot timeout detaches the poisoned debugger session before a queued evaluation reattaches and sends its first command. - The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 42 tests with explicit runtime guest identity, including immediate open, monotonic short-deadline handling, deadline-clamped visibility polling, and snapshot rejection after a server-epoch replacement. - Desktop, server, contracts, and scripts typechecks completed without type errors. The web typecheck currently reaches only three `RegistryContext` typing errors that are also present on `upstream/main`. diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index 22de1c31804..7e85d4227bd 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -7,6 +7,8 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { previewBridge } from "~/components/preview/previewBridge"; import { usePreviewBridge } from "~/components/preview/usePreviewBridge"; import { cn } from "~/lib/utils"; +import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; +import { selectThreadRightPanelState, useRightPanelStore } from "~/rightPanelStore"; import { resolveBrowserSurfaceBackgroundCaptureRect, @@ -21,7 +23,10 @@ import { import { BrowserDeviceToolbar } from "./BrowserDeviceToolbar"; import { BrowserViewportResizeHandles } from "./BrowserViewportResizeHandles"; import { acquireDesktopTab, type AcquiredDesktopTab } from "./desktopTabLifetime"; -import { resolveHostedBrowserWebviewWrapperStyle } from "./hostedBrowserWebviewStyle"; +import { + resolveHostedBrowserWebviewPresentation, + resolveHostedBrowserWebviewWrapperStyle, +} from "./hostedBrowserWebviewStyle"; import { usePreviewWebviewConfig } from "./previewWebviewConfigState"; import { useBrowserViewportResize } from "./useBrowserViewportResize"; import { @@ -64,6 +69,13 @@ export function HostedBrowserWebview(props: { const surface = useBrowserSurfaceStore( useShallow((state) => selectBrowserSurfaceRenderState(state, runtimeTabId)), ); + const selectedInRightPanel = useRightPanelStore((state) => { + const panel = selectThreadRightPanelState(state.byThreadKey, threadRef); + return panel.isOpen && panel.activeSurfaceId === `browser:${tabId}`; + }); + const selectedInMiniPlayer = usePreviewMiniPlayerStore( + (state) => selectThreadPreviewMiniPlayer(state.byThreadKey, threadRef)?.tabId === tabId, + ); const presentation = { backgroundCapture: surface.backgroundCapture, content: surface.content, @@ -154,8 +166,12 @@ export function HostedBrowserWebview(props: { }; }, [config, initialSrc, runtimeTabId, webviewGeneration]); - const active = presentation.visible && presentation.rect !== null; - const backgroundCapture = !active && presentation.backgroundCapture && presentation.rect !== null; + const { active, backgroundCapture } = resolveHostedBrowserWebviewPresentation({ + backgroundCaptureRequested: presentation.backgroundCapture, + hasRect: presentation.rect !== null, + selected: selectedInRightPanel || selectedInMiniPlayer, + surfaceVisible: presentation.visible, + }); const lastRect = presentation.rect; const normalizedZoomFactor = Number.isFinite(zoomFactor) && zoomFactor > 0 ? zoomFactor : 1; const viewportWidth = viewport._tag === "fill" ? null : viewport.width; diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts index a34197dcb65..87bf1aa269f 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts @@ -4,9 +4,40 @@ import { BACKGROUND_CAPTURE_BROWSER_WEBVIEW_OPACITY, BACKGROUND_CAPTURE_BROWSER_WEBVIEW_Z_INDEX, HIDDEN_BROWSER_WEBVIEW_OFFSET, + resolveHostedBrowserWebviewPresentation, resolveHostedBrowserWebviewWrapperStyle, } from "./hostedBrowserWebviewStyle"; +describe("resolveHostedBrowserWebviewPresentation", () => { + it("stages a background capture when visibility is stale after selection changes", () => { + expect( + resolveHostedBrowserWebviewPresentation({ + backgroundCaptureRequested: true, + hasRect: true, + selected: false, + surfaceVisible: true, + }), + ).toEqual({ + active: false, + backgroundCapture: true, + }); + }); + + it("keeps the selected visible surface in the foreground during a capture request", () => { + expect( + resolveHostedBrowserWebviewPresentation({ + backgroundCaptureRequested: true, + hasRect: true, + selected: true, + surfaceVisible: true, + }), + ).toEqual({ + active: true, + backgroundCapture: false, + }); + }); +}); + describe("resolveHostedBrowserWebviewWrapperStyle", () => { it("places an active webview on its presented surface", () => { expect( diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.ts index 07800e3e5fb..b322b39675d 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.ts @@ -21,6 +21,22 @@ export const HIDDEN_BROWSER_WEBVIEW_OFFSET = -100_000; export const BACKGROUND_CAPTURE_BROWSER_WEBVIEW_Z_INDEX = 31; export const BACKGROUND_CAPTURE_BROWSER_WEBVIEW_OPACITY = 0.001; +export function resolveHostedBrowserWebviewPresentation(input: { + readonly backgroundCaptureRequested: boolean; + readonly hasRect: boolean; + readonly selected: boolean; + readonly surfaceVisible: boolean; +}): { + readonly active: boolean; + readonly backgroundCapture: boolean; +} { + const active = input.selected && input.surfaceVisible && input.hasRect; + return { + active, + backgroundCapture: !active && input.backgroundCaptureRequested && input.hasRect, + }; +} + export function resolveHostedBrowserWebviewWrapperStyle(input: { readonly active: boolean; readonly backgroundCapture: boolean; From 54c97998e6af58403137c908faf94cd22848c378 Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Thu, 30 Jul 2026 13:33:47 +0100 Subject: [PATCH 33/43] fix(worktree): restore upstream legend list patch --- apps/mobile/package.json | 2 +- ...2.0.patch => @legendapp__list@3.3.3.patch} | 244 ++++++++++++------ pnpm-lock.yaml | 36 ++- pnpm-workspace.yaml | 2 +- 4 files changed, 185 insertions(+), 99 deletions(-) rename patches/{@legendapp__list@3.2.0.patch => @legendapp__list@3.3.3.patch} (82%) diff --git a/apps/mobile/package.json b/apps/mobile/package.json index c99351547be..9a5e64aa46f 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -49,7 +49,7 @@ "@expo-google-fonts/dm-sans": "^0.4.2", "@expo/metro-runtime": "~56.0.15", "@expo/ui": "~56.0.18", - "@legendapp/list": "3.2.0", + "@legendapp/list": "3.3.3", "@noble/curves": "catalog:", "@noble/hashes": "catalog:", "@pierre/diffs": "catalog:", diff --git a/patches/@legendapp__list@3.2.0.patch b/patches/@legendapp__list@3.3.3.patch similarity index 82% rename from patches/@legendapp__list@3.2.0.patch rename to patches/@legendapp__list@3.3.3.patch index 8059fbb5f4e..4fa135d5aa0 100644 --- a/patches/@legendapp__list@3.2.0.patch +++ b/patches/@legendapp__list@3.3.3.patch @@ -1,8 +1,8 @@ diff --git a/keyboard.d.ts b/keyboard.d.ts -index 5a115ea..2c65d31 100644 +index 7bc3bb8..75ec120 100644 --- a/keyboard.d.ts +++ b/keyboard.d.ts -@@ -269,7 +269,7 @@ type KeyboardChatComposerInsetListRef = { +@@ -277,7 +277,7 @@ type KeyboardChatComposerInsetListRef = { type KeyboardChatComposerRef = { current: Pick | null; }; @@ -11,7 +11,7 @@ index 5a115ea..2c65d31 100644 contentInsetEndAdjustment: SharedValue; onComposerLayout: (event: LayoutChangeEvent) => void; }; -@@ -278,8 +278,10 @@ declare function useKeyboardScrollToEnd({ freeze: freezeProp, listRef }: UseKeyb +@@ -286,8 +286,10 @@ declare function useKeyboardScrollToEnd({ freeze: freezeProp, listRef }: UseKeyb scrollMessageToEnd: ({ animated, closeKeyboard }: ScrollMessageToEndOptions) => Promise; }; declare const KeyboardAwareLegendList: (props: Omit, "anchoredEndSpace" | "contentInsetEndAdjustment" | "renderScrollComponent"> & KeyboardChatScrollViewPropsUnique & { @@ -187,10 +187,10 @@ index c1dd270..cb0d142 100644 renderScrollComponent: memoList, ...rest diff --git a/react-native.d.ts b/react-native.d.ts -index 72d3f59..435a5fc 100644 +index 8204015..cdeaab7 100644 --- a/react-native.d.ts +++ b/react-native.d.ts -@@ -284,6 +284,12 @@ interface LegendListSpecificProps { +@@ -293,6 +293,12 @@ interface LegendListSpecificProps { * The adjustment is also rendered as real content padding so the browser scroll range includes it. */ contentInsetEndAdjustment?: number; @@ -204,10 +204,10 @@ index 72d3f59..435a5fc 100644 * Number of columns to render items in. * @default 1 diff --git a/react-native.js b/react-native.js -index 8d4ff89..003b2f7 100644 +index 229f09a..2a1ceb6 100644 --- a/react-native.js +++ b/react-native.js -@@ -1195,7 +1195,7 @@ function setInitialRenderState(ctx, { +@@ -930,7 +930,7 @@ function setInitialRenderState(ctx, { if (didInitialScroll) { state.didFinishInitialScroll = true; } @@ -215,8 +215,8 @@ index 8d4ff89..003b2f7 100644 + const isReadyToRender = Boolean(state.didContainersLayout && state.didFinishInitialScroll && !state.insetEndRevealHold); if (isReadyToRender && !peek$(ctx, "readyToRender")) { set$(ctx, "readyToRender", true); - setAdaptiveRender(ctx, "normal"); -@@ -1480,18 +1480,23 @@ function calculateOffsetWithOffsetPosition(ctx, offsetParam, params) { + setAdaptiveRender(ctx, "normal", "ready"); +@@ -1259,18 +1259,23 @@ function calculateOffsetWithOffsetPosition(ctx, offsetParam, params) { } // src/core/clampScrollOffset.ts @@ -242,7 +242,7 @@ index 8d4ff89..003b2f7 100644 return clampedOffset; } -@@ -1626,10 +1631,10 @@ function checkFinishedScrollFrame(ctx) { +@@ -1406,10 +1411,10 @@ function checkFinishedScrollFrame(ctx) { finishScrollTo(ctx); } } @@ -255,7 +255,7 @@ index 8d4ff89..003b2f7 100644 x: ctx.state.props.horizontal ? offset : 0, y: ctx.state.props.horizontal ? 0 : offset }); -@@ -1676,7 +1681,10 @@ function checkFinishedScrollFallback(ctx) { +@@ -1456,7 +1461,10 @@ function checkFinishedScrollFallback(ctx) { }); scheduleFallbackCheck(SILENT_INITIAL_SCROLL_RETRY_DELAY_MS); } else if (shouldRetryUnalignedEndScroll) { @@ -267,10 +267,10 @@ index 8d4ff89..003b2f7 100644 scheduleFallbackCheck(100); } else if (shouldFinishZeroTarget || shouldFinishAfterObservedScroll || canFinishInitialScrollWithoutNativeProgress || canFinishAfterSilentNativeDispatch || numChecks > maxChecks) { finishScrollTo(ctx); -@@ -1737,9 +1745,18 @@ function doMaintainScrollAtEnd(ctx) { +@@ -1517,9 +1525,18 @@ function doMaintainScrollAtEnd(ctx) { } - state.pendingMaintainScrollAtEnd = false; if (shouldMaintainScrollAtEnd) { + state.pendingMaintainScrollAtEnd = false; + const maintainAnchoredEndSpace = state.props.anchoredEndSpace; + const maintainAnchorIndex = maintainAnchoredEndSpace == null ? void 0 : maintainAnchoredEndSpace.anchorIndex; + if (maintainAnchorIndex !== void 0 && maintainAnchorIndex >= 0 && maintainAnchorIndex < state.props.data.length && !areKnownOrFixedItemSizesAvailable(ctx, maintainAnchorIndex, state.props.data.length - 1)) { @@ -287,7 +287,7 @@ index 8d4ff89..003b2f7 100644 } if (!state.maintainingScrollAtEnd) { const pendingState = maintainScrollAtEnd.animated ? "pending-animated" : "pending-instant"; -@@ -1759,9 +1776,18 @@ function doMaintainScrollAtEnd(ctx) { +@@ -1539,9 +1556,18 @@ function doMaintainScrollAtEnd(ctx) { y: 0 }); } else { @@ -309,7 +309,18 @@ index 8d4ff89..003b2f7 100644 } setTimeout( () => { -@@ -1888,7 +1914,9 @@ function getPredictedNativeClamp(state, unresolvedAmount, totalSize) { +@@ -1571,6 +1597,10 @@ function doMaintainScrollAtEnd(ctx) { + function requestAdjust(ctx, positionDiff, dataChanged) { + const state = ctx.state; + if (Math.abs(positionDiff) > 0.1) { ++ // Timestamp for ReanimatedPositionView: repositions caused by an MVCP ++ // size adjustment are already compensated by a contentOffset shift, so ++ // animating them would make rows visibly lurch and slide back. ++ state.lastMVCPAdjustTime = Date.now(); + const needsScrollWorkaround = Platform.OS === "android" && !IsNewArchitecture && dataChanged && state.scroll <= positionDiff; + const doit = () => { + if (needsScrollWorkaround) { +@@ -1674,7 +1704,9 @@ function getPredictedNativeClamp(state, unresolvedAmount, totalSize) { if (Math.abs(unresolvedAmount) <= MVCP_POSITION_EPSILON) { return 0; } @@ -320,7 +331,7 @@ index 8d4ff89..003b2f7 100644 const clampDelta = maxScroll - state.scroll; if (unresolvedAmount < 0) { return Math.max(unresolvedAmount, Math.min(0, clampDelta)); -@@ -1950,7 +1978,7 @@ function resolvePendingNativeMVCPAdjust(ctx, newScroll) { +@@ -1736,7 +1768,7 @@ function resolvePendingNativeMVCPAdjust(ctx, newScroll) { settlePendingNativeMVCPAdjust(ctx, remainingAfterManual, nativeDelta); return true; } @@ -329,7 +340,7 @@ index 8d4ff89..003b2f7 100644 const distanceToClamp = Math.abs(newScroll - expectedNativeClampScroll); const isAtExpectedNativeClamp = distanceToClamp <= NATIVE_END_CLAMP_EPSILON; if (isAtExpectedNativeClamp) { -@@ -2083,7 +2111,7 @@ function prepareMVCP(ctx, dataChanged) { +@@ -1869,7 +1901,7 @@ function prepareMVCP(ctx, dataChanged) { if (diff > 0) { diff = Math.max(0, totalSize - state.scroll - state.scrollLength); } else { @@ -338,7 +349,7 @@ index 8d4ff89..003b2f7 100644 state.scroll = maxScroll; state.scrollPending = maxScroll; diff = 0; -@@ -2374,8 +2402,121 @@ function scrollToIndex(ctx, { +@@ -2274,8 +2306,121 @@ function scrollToIndex(ctx, { } // src/core/initialScroll.ts @@ -460,7 +471,7 @@ index 8d4ff89..003b2f7 100644 const requestedIndex = target.index; const index = requestedIndex !== void 0 ? clampScrollIndex(requestedIndex, ctx.state.props.data.length) : void 0; const itemSize = getItemSizeAtIndex(ctx, index); -@@ -2804,7 +2945,9 @@ function clearFinishedBootstrapInitialScrollTargetIfMovedAway(ctx) { +@@ -2704,7 +2849,9 @@ function clearFinishedBootstrapInitialScrollTargetIfMovedAway(ctx) { return; } if (didFinishedInitialScrollMoveAwayFromTarget(ctx, initialScroll)) { @@ -471,7 +482,7 @@ index 8d4ff89..003b2f7 100644 if (!shouldKeepEndTargetAlive) { if (shouldPreserveInitialScrollForFooterLayout(initialScroll)) { clearPendingInitialScrollFooterLayout(ctx, { -@@ -4646,7 +4789,8 @@ function maybeUpdateAnchoredEndSpace(ctx) { +@@ -4637,7 +4784,8 @@ function maybeUpdateAnchoredEndSpace(ctx) { } contentBelowAnchor += footerSize + stylePaddingBottom; isReady = !hasUnknownTailSize; @@ -481,7 +492,7 @@ index 8d4ff89..003b2f7 100644 } else if (anchorIndex >= 0) { isReady = false; } -@@ -4664,6 +4808,12 @@ function maybeUpdateAnchoredEndSpace(ctx) { +@@ -4655,6 +4803,12 @@ function maybeUpdateAnchoredEndSpace(ctx) { updateScroll(ctx, state.scroll, true); } (_b = anchoredEndSpace == null ? void 0 : anchoredEndSpace.onReady) == null ? void 0 : _b.call(anchoredEndSpace, { anchorIndex: nextAnchorIndex, anchorKey: nextAnchorKey, size: nextSize }); @@ -494,7 +505,7 @@ index 8d4ff89..003b2f7 100644 } return nextSize; } -@@ -6462,6 +6612,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded +@@ -6960,6 +7114,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded dataVersion, drawDistance = 250, contentInsetEndAdjustment, @@ -502,15 +513,15 @@ index 8d4ff89..003b2f7 100644 estimatedItemSize = 100, estimatedListSize, extraData, -@@ -6492,6 +6643,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded +@@ -6990,6 +7145,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded onLayout: onLayoutProp, onLoad, onMomentumScrollEnd, + onScrollBeginDrag, onRefresh, onScroll: onScrollProp, - onStartReached, -@@ -6577,7 +6729,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded + onScrollBeginDrag, +@@ -7076,7 +7232,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded const combinedRef = useCombinedRef(refScroller, refScrollView); const keyExtractor = keyExtractorProp != null ? keyExtractorProp : ((_item, index) => index.toString()); const stickyHeaderIndices = stickyHeaderIndicesProp; @@ -519,15 +530,15 @@ index 8d4ff89..003b2f7 100644 const previousContentInsetEndAdjustmentRef = React2.useRef(contentInsetEndAdjustmentResolved); const alwaysRenderIndices = React2.useMemo(() => { const indices = getAlwaysRenderIndices(alwaysRender, dataProp, keyExtractor, anchoredEndSpace == null ? void 0 : anchoredEndSpace.anchorIndex); -@@ -6710,6 +6862,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded +@@ -7215,6 +7371,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded contentContainerAlignItems: contentContainerStyle.alignItems, contentInset, contentInsetEndAdjustment: contentInsetEndAdjustmentResolved, + contentInsetStartAdjustment, data: dataProp, + dataKey, dataVersion, - drawDistance, -@@ -6789,6 +6942,13 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded +@@ -7303,6 +7460,13 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded return void 0; } const resolvedOffset = (_a4 = initialScroll.contentOffset) != null ? _a4 : resolveInitialScrollOffset(ctx, initialScroll); @@ -541,20 +552,15 @@ index 8d4ff89..003b2f7 100644 return usesBootstrapInitialScroll && ((_b2 = state.initialScrollSession) == null ? void 0 : _b2.kind) === "bootstrap" && Platform.OS === "web" ? void 0 : resolvedOffset; }, [usesBootstrapInitialScroll]); React2.useLayoutEffect(() => { -@@ -6995,6 +7155,12 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded - onMomentumScrollEnd(event); - } - }, -+ onScrollBeginDrag: (event) => { +@@ -7526,6 +7690,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded + onScroll: (event) => onScroll(ctx, event), + onScrollBeginDrag: (event) => { + var _a4, _b2; + ctx.state.didUserDrag = true; -+ if (onScrollBeginDrag) { -+ onScrollBeginDrag(event); -+ } -+ }, - onScroll: (event) => onScroll(ctx, event) - }), - [] -@@ -7019,6 +7185,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded + prepareReachedEdgeForNextUserScroll(ctx); + (_b2 = (_a4 = state.props).onScrollBeginDrag) == null ? void 0 : _b2.call(_a4, event); + }, +@@ -7555,6 +7720,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded onLayout, onLayoutFooter, onMomentumScrollEnd: fns.onMomentumScrollEnd, @@ -563,10 +569,10 @@ index 8d4ff89..003b2f7 100644 recycleItems, refreshControl: refreshControlElement ? stylePaddingTopState > 0 ? React2__namespace.cloneElement(refreshControlElement, { diff --git a/react-native.mjs b/react-native.mjs -index 2e96ca7..fc88d4b 100644 +index c2e0f38..5313086 100644 --- a/react-native.mjs +++ b/react-native.mjs -@@ -1174,7 +1174,7 @@ function setInitialRenderState(ctx, { +@@ -909,7 +909,7 @@ function setInitialRenderState(ctx, { if (didInitialScroll) { state.didFinishInitialScroll = true; } @@ -574,8 +580,8 @@ index 2e96ca7..fc88d4b 100644 + const isReadyToRender = Boolean(state.didContainersLayout && state.didFinishInitialScroll && !state.insetEndRevealHold); if (isReadyToRender && !peek$(ctx, "readyToRender")) { set$(ctx, "readyToRender", true); - setAdaptiveRender(ctx, "normal"); -@@ -1459,18 +1459,23 @@ function calculateOffsetWithOffsetPosition(ctx, offsetParam, params) { + setAdaptiveRender(ctx, "normal", "ready"); +@@ -1238,18 +1238,23 @@ function calculateOffsetWithOffsetPosition(ctx, offsetParam, params) { } // src/core/clampScrollOffset.ts @@ -601,7 +607,7 @@ index 2e96ca7..fc88d4b 100644 return clampedOffset; } -@@ -1605,10 +1610,10 @@ function checkFinishedScrollFrame(ctx) { +@@ -1385,10 +1390,10 @@ function checkFinishedScrollFrame(ctx) { finishScrollTo(ctx); } } @@ -614,7 +620,7 @@ index 2e96ca7..fc88d4b 100644 x: ctx.state.props.horizontal ? offset : 0, y: ctx.state.props.horizontal ? 0 : offset }); -@@ -1655,7 +1660,10 @@ function checkFinishedScrollFallback(ctx) { +@@ -1435,7 +1440,10 @@ function checkFinishedScrollFallback(ctx) { }); scheduleFallbackCheck(SILENT_INITIAL_SCROLL_RETRY_DELAY_MS); } else if (shouldRetryUnalignedEndScroll) { @@ -626,10 +632,10 @@ index 2e96ca7..fc88d4b 100644 scheduleFallbackCheck(100); } else if (shouldFinishZeroTarget || shouldFinishAfterObservedScroll || canFinishInitialScrollWithoutNativeProgress || canFinishAfterSilentNativeDispatch || numChecks > maxChecks) { finishScrollTo(ctx); -@@ -1716,9 +1724,18 @@ function doMaintainScrollAtEnd(ctx) { +@@ -1496,9 +1504,18 @@ function doMaintainScrollAtEnd(ctx) { } - state.pendingMaintainScrollAtEnd = false; if (shouldMaintainScrollAtEnd) { + state.pendingMaintainScrollAtEnd = false; + const maintainAnchoredEndSpace = state.props.anchoredEndSpace; + const maintainAnchorIndex = maintainAnchoredEndSpace == null ? void 0 : maintainAnchoredEndSpace.anchorIndex; + if (maintainAnchorIndex !== void 0 && maintainAnchorIndex >= 0 && maintainAnchorIndex < state.props.data.length && !areKnownOrFixedItemSizesAvailable(ctx, maintainAnchorIndex, state.props.data.length - 1)) { @@ -646,7 +652,7 @@ index 2e96ca7..fc88d4b 100644 } if (!state.maintainingScrollAtEnd) { const pendingState = maintainScrollAtEnd.animated ? "pending-animated" : "pending-instant"; -@@ -1738,9 +1755,18 @@ function doMaintainScrollAtEnd(ctx) { +@@ -1518,9 +1535,18 @@ function doMaintainScrollAtEnd(ctx) { y: 0 }); } else { @@ -668,7 +674,18 @@ index 2e96ca7..fc88d4b 100644 } setTimeout( () => { -@@ -1867,7 +1893,9 @@ function getPredictedNativeClamp(state, unresolvedAmount, totalSize) { +@@ -1550,6 +1576,10 @@ function doMaintainScrollAtEnd(ctx) { + function requestAdjust(ctx, positionDiff, dataChanged) { + const state = ctx.state; + if (Math.abs(positionDiff) > 0.1) { ++ // Timestamp for ReanimatedPositionView: repositions caused by an MVCP ++ // size adjustment are already compensated by a contentOffset shift, so ++ // animating them would make rows visibly lurch and slide back. ++ state.lastMVCPAdjustTime = Date.now(); + const needsScrollWorkaround = Platform.OS === "android" && !IsNewArchitecture && dataChanged && state.scroll <= positionDiff; + const doit = () => { + if (needsScrollWorkaround) { +@@ -1653,7 +1683,9 @@ function getPredictedNativeClamp(state, unresolvedAmount, totalSize) { if (Math.abs(unresolvedAmount) <= MVCP_POSITION_EPSILON) { return 0; } @@ -679,7 +696,7 @@ index 2e96ca7..fc88d4b 100644 const clampDelta = maxScroll - state.scroll; if (unresolvedAmount < 0) { return Math.max(unresolvedAmount, Math.min(0, clampDelta)); -@@ -1929,7 +1957,7 @@ function resolvePendingNativeMVCPAdjust(ctx, newScroll) { +@@ -1715,7 +1747,7 @@ function resolvePendingNativeMVCPAdjust(ctx, newScroll) { settlePendingNativeMVCPAdjust(ctx, remainingAfterManual, nativeDelta); return true; } @@ -688,7 +705,7 @@ index 2e96ca7..fc88d4b 100644 const distanceToClamp = Math.abs(newScroll - expectedNativeClampScroll); const isAtExpectedNativeClamp = distanceToClamp <= NATIVE_END_CLAMP_EPSILON; if (isAtExpectedNativeClamp) { -@@ -2062,7 +2090,7 @@ function prepareMVCP(ctx, dataChanged) { +@@ -1848,7 +1880,7 @@ function prepareMVCP(ctx, dataChanged) { if (diff > 0) { diff = Math.max(0, totalSize - state.scroll - state.scrollLength); } else { @@ -697,7 +714,7 @@ index 2e96ca7..fc88d4b 100644 state.scroll = maxScroll; state.scrollPending = maxScroll; diff = 0; -@@ -2353,8 +2381,121 @@ function scrollToIndex(ctx, { +@@ -2253,8 +2285,121 @@ function scrollToIndex(ctx, { } // src/core/initialScroll.ts @@ -819,7 +836,7 @@ index 2e96ca7..fc88d4b 100644 const requestedIndex = target.index; const index = requestedIndex !== void 0 ? clampScrollIndex(requestedIndex, ctx.state.props.data.length) : void 0; const itemSize = getItemSizeAtIndex(ctx, index); -@@ -2783,7 +2924,9 @@ function clearFinishedBootstrapInitialScrollTargetIfMovedAway(ctx) { +@@ -2683,7 +2828,9 @@ function clearFinishedBootstrapInitialScrollTargetIfMovedAway(ctx) { return; } if (didFinishedInitialScrollMoveAwayFromTarget(ctx, initialScroll)) { @@ -830,7 +847,7 @@ index 2e96ca7..fc88d4b 100644 if (!shouldKeepEndTargetAlive) { if (shouldPreserveInitialScrollForFooterLayout(initialScroll)) { clearPendingInitialScrollFooterLayout(ctx, { -@@ -4625,7 +4768,8 @@ function maybeUpdateAnchoredEndSpace(ctx) { +@@ -4616,7 +4763,8 @@ function maybeUpdateAnchoredEndSpace(ctx) { } contentBelowAnchor += footerSize + stylePaddingBottom; isReady = !hasUnknownTailSize; @@ -840,7 +857,7 @@ index 2e96ca7..fc88d4b 100644 } else if (anchorIndex >= 0) { isReady = false; } -@@ -4643,6 +4787,12 @@ function maybeUpdateAnchoredEndSpace(ctx) { +@@ -4634,6 +4782,12 @@ function maybeUpdateAnchoredEndSpace(ctx) { updateScroll(ctx, state.scroll, true); } (_b = anchoredEndSpace == null ? void 0 : anchoredEndSpace.onReady) == null ? void 0 : _b.call(anchoredEndSpace, { anchorIndex: nextAnchorIndex, anchorKey: nextAnchorKey, size: nextSize }); @@ -853,7 +870,7 @@ index 2e96ca7..fc88d4b 100644 } return nextSize; } -@@ -6441,6 +6591,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded +@@ -6939,6 +7093,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded dataVersion, drawDistance = 250, contentInsetEndAdjustment, @@ -861,15 +878,7 @@ index 2e96ca7..fc88d4b 100644 estimatedItemSize = 100, estimatedListSize, extraData, -@@ -6471,6 +6622,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded - onLayout: onLayoutProp, - onLoad, - onMomentumScrollEnd, -+ onScrollBeginDrag, - onRefresh, - onScroll: onScrollProp, - onStartReached, -@@ -6556,7 +6708,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded +@@ -7055,7 +7210,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded const combinedRef = useCombinedRef(refScroller, refScrollView); const keyExtractor = keyExtractorProp != null ? keyExtractorProp : ((_item, index) => index.toString()); const stickyHeaderIndices = stickyHeaderIndicesProp; @@ -878,15 +887,15 @@ index 2e96ca7..fc88d4b 100644 const previousContentInsetEndAdjustmentRef = useRef(contentInsetEndAdjustmentResolved); const alwaysRenderIndices = useMemo(() => { const indices = getAlwaysRenderIndices(alwaysRender, dataProp, keyExtractor, anchoredEndSpace == null ? void 0 : anchoredEndSpace.anchorIndex); -@@ -6689,6 +6841,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded +@@ -7194,6 +7349,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded contentContainerAlignItems: contentContainerStyle.alignItems, contentInset, contentInsetEndAdjustment: contentInsetEndAdjustmentResolved, + contentInsetStartAdjustment, data: dataProp, + dataKey, dataVersion, - drawDistance, -@@ -6768,6 +6921,13 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded +@@ -7282,6 +7438,13 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded return void 0; } const resolvedOffset = (_a4 = initialScroll.contentOffset) != null ? _a4 : resolveInitialScrollOffset(ctx, initialScroll); @@ -900,20 +909,15 @@ index 2e96ca7..fc88d4b 100644 return usesBootstrapInitialScroll && ((_b2 = state.initialScrollSession) == null ? void 0 : _b2.kind) === "bootstrap" && Platform.OS === "web" ? void 0 : resolvedOffset; }, [usesBootstrapInitialScroll]); useLayoutEffect(() => { -@@ -6974,6 +7134,12 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded - onMomentumScrollEnd(event); - } - }, -+ onScrollBeginDrag: (event) => { +@@ -7505,6 +7668,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded + onScroll: (event) => onScroll(ctx, event), + onScrollBeginDrag: (event) => { + var _a4, _b2; + ctx.state.didUserDrag = true; -+ if (onScrollBeginDrag) { -+ onScrollBeginDrag(event); -+ } -+ }, - onScroll: (event) => onScroll(ctx, event) - }), - [] -@@ -6998,6 +7164,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded + prepareReachedEdgeForNextUserScroll(ctx); + (_b2 = (_a4 = state.props).onScrollBeginDrag) == null ? void 0 : _b2.call(_a4, event); + }, +@@ -7534,6 +7698,7 @@ var LegendListInner = typedForwardRef(function LegendListInner2(props, forwarded onLayout, onLayoutFooter, onMomentumScrollEnd: fns.onMomentumScrollEnd, @@ -922,10 +926,10 @@ index 2e96ca7..fc88d4b 100644 recycleItems, refreshControl: refreshControlElement ? stylePaddingTopState > 0 ? React2.cloneElement(refreshControlElement, { diff --git a/reanimated.d.ts b/reanimated.d.ts -index 7e2d11f..d5b0d66 100644 +index 940da28..28dccbe 100644 --- a/reanimated.d.ts +++ b/reanimated.d.ts -@@ -285,6 +285,12 @@ interface LegendListSpecificProps { +@@ -294,6 +294,12 @@ interface LegendListSpecificProps { * The adjustment is also rendered as real content padding so the browser scroll range includes it. */ contentInsetEndAdjustment?: number; @@ -938,3 +942,73 @@ index 7e2d11f..d5b0d66 100644 /** * Number of columns to render items in. * @default 1 +diff --git a/reanimated.js b/reanimated.js +index f1265fa..16dcef0 100644 +--- a/reanimated.js ++++ b/reanimated.js +@@ -116,7 +116,7 @@ var ReanimatedPositionView = typedMemo(function ReanimatedPositionViewComponent( + const [positionValue = POSITION_OUT_OF_VIEW] = useArr$([`containerPosition${id}`]); + const prevItemKeyRef = React__namespace.useRef(void 0); + let shouldSkipTransitionForRecycleReuse = false; +- if (recycleItems && layoutTransition) { ++ if (layoutTransition) { + const itemKeySignal = `containerItemKey${id}`; + const itemKey = peek$(ctx, itemKeySignal); + shouldSkipTransitionForRecycleReuse = itemKey !== void 0 && prevItemKeyRef.current !== void 0 && prevItemKeyRef.current !== itemKey; +@@ -130,10 +130,20 @@ var ReanimatedPositionView = typedMemo(function ReanimatedPositionViewComponent( + () => [style, horizontal ? { left: positionValue } : { top: positionValue }], + [horizontal, positionValue, style] + ); ++ // Two kinds of repositions must not animate: (1) MVCP size adjustments, ++ // which are compensated by an equal contentOffset shift so the row should ++ // not visibly move — animating turns the invisible correction into a ++ // lurch-and-settle; (2) any reposition while the user is actively ++ // scrolling, where rows shifting under the finger reads as jank. The ++ // transition exists to smooth at-rest shifts (streaming text, work-log ++ // folds), so gate it to at-rest moments. ++ const now = Date.now(); ++ const isMVCPReposition = now - (ctx.state.lastMVCPAdjustTime || 0) < 300 || ++ now - (ctx.state.lastNativeScrollTime || 0) < 300; + return /* @__PURE__ */ React__namespace.createElement( + Reanimated__default.default.View, + { +- layout: shouldSkipTransitionForRecycleReuse ? void 0 : layoutTransition, ++ layout: shouldSkipTransitionForRecycleReuse || isMVCPReposition ? void 0 : layoutTransition, + ref: refView, + style: viewStyle, + ...rest +diff --git a/reanimated.mjs b/reanimated.mjs +index 29a00d5..9c25ec5 100644 +--- a/reanimated.mjs ++++ b/reanimated.mjs +@@ -92,7 +92,7 @@ var ReanimatedPositionView = typedMemo(function ReanimatedPositionViewComponent( + const [positionValue = POSITION_OUT_OF_VIEW] = useArr$([`containerPosition${id}`]); + const prevItemKeyRef = React.useRef(void 0); + let shouldSkipTransitionForRecycleReuse = false; +- if (recycleItems && layoutTransition) { ++ if (layoutTransition) { + const itemKeySignal = `containerItemKey${id}`; + const itemKey = peek$(ctx, itemKeySignal); + shouldSkipTransitionForRecycleReuse = itemKey !== void 0 && prevItemKeyRef.current !== void 0 && prevItemKeyRef.current !== itemKey; +@@ -106,10 +106,20 @@ var ReanimatedPositionView = typedMemo(function ReanimatedPositionViewComponent( + () => [style, horizontal ? { left: positionValue } : { top: positionValue }], + [horizontal, positionValue, style] + ); ++ // Two kinds of repositions must not animate: (1) MVCP size adjustments, ++ // which are compensated by an equal contentOffset shift so the row should ++ // not visibly move — animating turns the invisible correction into a ++ // lurch-and-settle; (2) any reposition while the user is actively ++ // scrolling, where rows shifting under the finger reads as jank. The ++ // transition exists to smooth at-rest shifts (streaming text, work-log ++ // folds), so gate it to at-rest moments. ++ const now = Date.now(); ++ const isMVCPReposition = now - (ctx.state.lastMVCPAdjustTime || 0) < 300 || ++ now - (ctx.state.lastNativeScrollTime || 0) < 300; + return /* @__PURE__ */ React.createElement( + Reanimated.View, + { +- layout: shouldSkipTransitionForRecycleReuse ? void 0 : layoutTransition, ++ layout: shouldSkipTransitionForRecycleReuse || isMVCPReposition ? void 0 : layoutTransition, + ref: refView, + style: viewStyle, + ...rest diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c46d2c6a4dc..816846ab529 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -74,7 +74,7 @@ patchedDependencies: '@effect/vitest@4.0.0-beta.102': a607339aab944136a084a05f159aa0f5a69776934d4b835dab8f9992f1c13425 '@expo/metro-config@56.0.14': 8cb08b5bb7051ed9d2dbe46a2c293c5a1e17f1bd6ddf30de27909e18c921ff46 '@ff-labs/fff-node@0.9.4': 2b16019ce7ab61aec6478dd02f79ef468cc1d5c51e9d00764f7d2ab8167210c8 - '@legendapp/list@3.2.0': 9203ec3db86574a31e157a265cefd8997b259cc420a0c4c61c580c80ed8cf78d + '@legendapp/list@3.3.3': d162d67b73933ab077d00627cdf52b18ea88b28c4fae4e8340a854df25026f09 '@pierre/diffs@1.3.0-beta.10': 7ef7cb0cbabb17c15cdb137554068b36f15f6f5265e73fb51452aa3380db91aa '@react-native-menu/menu@2.0.0': 5ea3ae4bf1d9baf5443b65c269bb09621c27a68d556f713778f37b1e8d46aaae '@react-navigation/native-stack@7.17.6': c7fc101b78d434904425e5a24c22fb0042298dec6f807250486e784f3c717273 @@ -212,8 +212,8 @@ importers: specifier: ~56.0.18 version: 56.0.18(3cdc0dde9f93166d952f1e1bd0cb25c0) '@legendapp/list': - specifier: 3.2.0 - version: 3.2.0(patch_hash=9203ec3db86574a31e157a265cefd8997b259cc420a0c4c61c580c80ed8cf78d)(react-dom@19.2.3(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) + specifier: 3.3.3 + version: 3.3.3(patch_hash=d162d67b73933ab077d00627cdf52b18ea88b28c4fae4e8340a854df25026f09)(react-dom@19.2.3(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3) '@noble/curves': specifier: 'catalog:' version: 1.9.1 @@ -545,7 +545,7 @@ importers: version: 0.9.0 '@legendapp/list': specifier: 3.2.0 - version: 3.2.0(patch_hash=9203ec3db86574a31e157a265cefd8997b259cc420a0c4c61c580c80ed8cf78d)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + version: 3.2.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6) '@lexical/react': specifier: ^0.41.0 version: 0.41.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(yjs@13.6.31) @@ -2975,6 +2975,18 @@ packages: react-native: optional: true + '@legendapp/list@3.3.3': + resolution: {integrity: sha512-p3g4xG6f//s4XQKhuus2189GCQgOHEIbJXHePqeDxj+6UQQQyij4YBjyArNSCgqoP0c03sxDPSOuCFB128Ql6g==} + peerDependencies: + react: '*' + react-dom: '*' + react-native: '*' + peerDependenciesMeta: + react-dom: + optional: true + react-native: + optional: true + '@lexical/clipboard@0.41.0': resolution: {integrity: sha512-Ex5lPkb4NBBX1DCPzOAIeHBJFH1bJcmATjREaqpnTfxCbuOeQkt44wchezUA0oDl+iAxNZ3+pLLWiUju9icoSA==} @@ -12982,7 +12994,14 @@ snapshots: dependencies: jsbi: 4.3.2 - '@legendapp/list@3.2.0(patch_hash=9203ec3db86574a31e157a265cefd8997b259cc420a0c4c61c580c80ed8cf78d)(react-dom@19.2.3(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)': + '@legendapp/list@3.2.0(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + dependencies: + react: 19.2.6 + use-sync-external-store: 1.6.0(react@19.2.6) + optionalDependencies: + react-dom: 19.2.6(react@19.2.6) + + '@legendapp/list@3.3.3(patch_hash=d162d67b73933ab077d00627cdf52b18ea88b28c4fae4e8340a854df25026f09)(react-dom@19.2.3(react@19.2.3))(react-native@0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))(react@19.2.3)': dependencies: react: 19.2.3 use-sync-external-store: 1.6.0(react@19.2.3) @@ -12990,13 +13009,6 @@ snapshots: react-dom: 19.2.3(react@19.2.3) react-native: 0.85.3(@babel/core@7.29.7)(@react-native/metro-config@0.85.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6) - '@legendapp/list@3.2.0(patch_hash=9203ec3db86574a31e157a265cefd8997b259cc420a0c4c61c580c80ed8cf78d)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': - dependencies: - react: 19.2.6 - use-sync-external-store: 1.6.0(react@19.2.6) - optionalDependencies: - react-dom: 19.2.6(react@19.2.6) - '@lexical/clipboard@0.41.0': dependencies: '@lexical/html': 0.41.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 70f61242053..d41e50e8784 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -127,7 +127,7 @@ patchedDependencies: "@effect/vitest@4.0.0-beta.102": patches/@effect__vitest@4.0.0-beta.102.patch "@expo/metro-config@56.0.14": patches/@expo%2Fmetro-config@56.0.14.patch "@ff-labs/fff-node@0.9.4": patches/@ff-labs__fff-node@0.9.4.patch - "@legendapp/list@3.2.0": patches/@legendapp__list@3.2.0.patch + "@legendapp/list@3.3.3": patches/@legendapp__list@3.3.3.patch "@pierre/diffs@1.3.0-beta.10": patches/@pierre%2Fdiffs@1.3.0-beta.10.patch "@react-native-menu/menu@2.0.0": patches/@react-native-menu__menu@2.0.0.patch "@react-navigation/native-stack@7.17.6": patches/@react-navigation%2Fnative-stack@7.17.6.patch From 926cf4bcd5a78375b81528b9a33c00e81120f327 Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Mon, 3 Aug 2026 15:19:45 +0100 Subject: [PATCH 34/43] Clamp preview input timeouts to host budget - Forward remaining renderer time into click, type, and wait calls - Cover delayed capture rejection after a presentation timeout - Refresh branch behavior and focused verification evidence --- BRANCH_DETAILS.md | 6 +- .../preview/PreviewAutomationHosts.tsx | 22 +++++++- .../previewAutomationPresentation.test.ts | 55 +++++++++++++++++++ .../preview/previewAutomationPresentation.ts | 3 + .../previewAutomationRequestConsumer.test.ts | 19 +++++++ .../previewAutomationRequestConsumer.ts | 9 +++ 6 files changed, 108 insertions(+), 6 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 7cec55fff0a..3568074d5fd 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -18,7 +18,7 @@ This branch does not add a second recording/PiP capture lifecycle or another hid Expected behavior: - Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout without making the execution budget shrink when the caller increases a short timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session while still holding an acquired control permit when a CDP command may be pending. Session removal and debugger teardown are atomic with respect to new session acquisition and bound to the exact acquired session, so late interruption or snapshot cleanup cannot detach a healthy replacement. Operations already queued on the retired semaphore detect that stale session and retry against its replacement. A request that times out while queued behind another action does not detach that action's shared debugger session. -- Click, type, and wait operations propagate their caller-supplied timeout into the desktop control-session boundary. Operations without a caller timeout retain the bounded desktop default. +- Click, type, and wait operations clamp their caller-supplied timeout to the remaining renderer host budget before entering the desktop control-session boundary. Operations without a caller timeout use the remaining bounded request budget rather than restarting the desktop default after renderer readiness work. - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Primary and fallback screenshot waits are clamped to the remaining control-session deadline, with budget reserved for fallback and result settlement, so a tight caller deadline can still return semantic data instead of being preempted by the outer session timeout. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, an actually timed-out CDP capture resets the session before releasing its control permit, queued work reattaches before issuing its first command, and a capture skipped before CDP runs leaves the healthy session attached. The semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. A persisted non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. - Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the server epoch replaces the runtime guest while staging is pending, the snapshot fails immediately with `PreviewAutomationTargetUnavailableError` instead of waiting on the stale staging marker. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. @@ -69,9 +69,9 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.t Current verification: -- The focused command above passed all 228 tests, including current-tab-only browser-surface selection, stale-visible background staging after selection changes, and serialized pairing submissions. +- The focused command above passed all 230 tests, including remaining-host-budget propagation for timeout-bearing desktop inputs, handled delayed capture rejection after a presentation timeout, current-tab-only browser-surface selection, stale-visible background staging after selection changes, and serialized pairing submissions. - The desktop manager's 35 focused tests additionally verify that a screenshot timeout detaches the poisoned debugger session before a queued evaluation reattaches and sends its first command. -- The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 42 tests with explicit runtime guest identity, including immediate open, monotonic short-deadline handling, deadline-clamped visibility polling, and snapshot rejection after a server-epoch replacement. +- The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 44 tests with explicit runtime guest identity, including immediate open, monotonic short-deadline handling, remaining desktop input budgets, deadline-clamped visibility polling, handled delayed capture rejection, and snapshot rejection after a server-epoch replacement. - Desktop, server, contracts, and scripts typechecks completed without type errors. The web typecheck currently reaches only three `RegistryContext` typing errors that are also present on `upstream/main`. - An isolated worktree `dev:desktop` using the user-data override paired successfully and stayed usable through repeated renderer/CDP inspection and raster capture. An archive worktree client loaded its seeded project against that host without the prior IndexedDB `VersionError` or host-interface disappearance. - An isolated web client on ports `5744`/`13784` completed first-navigation pairing, loaded the seeded Preview Reliability thread, and rendered the right-panel surface chooser. Its non-Electron Browser surface was unavailable as described under current limitations. diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index eb20c7dea2d..29cd0dd9fc4 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -67,6 +67,7 @@ import { import { createPreviewAutomationRequestConsumerAtom, previewAutomationExecutionBudget, + previewAutomationInputWithRemainingTimeout, previewAutomationRemainingBudget, } from "./previewAutomationRequestConsumer"; import { @@ -632,16 +633,26 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "click": { const ready = await requireReadyTab(); + const input = request.input as Parameters[1]; return await ready.bridge.automation.click( ready.runtimeTabId, - request.input as Parameters[1], + previewAutomationInputWithRemainingTimeout( + input, + request.timeoutMs, + remainingOperationBudget, + ), ); } case "type": { const ready = await requireReadyTab(); + const input = request.input as Parameters[1]; return await ready.bridge.automation.type( ready.runtimeTabId, - request.input as Parameters[1], + previewAutomationInputWithRemainingTimeout( + input, + request.timeoutMs, + remainingOperationBudget, + ), ); } case "press": { @@ -667,9 +678,14 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "waitFor": { const ready = await requireReadyTab(); + const input = request.input as Parameters[1]; return await ready.bridge.automation.waitFor( ready.runtimeTabId, - request.input as Parameters[1], + previewAutomationInputWithRemainingTimeout( + input, + request.timeoutMs, + remainingOperationBudget, + ), ); } case "recordingStart": { diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index 85caaae5eae..726c5e0c1ba 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -528,4 +528,59 @@ describe("preview automation presentation", () => { vi.useRealTimers(); } }); + + it("handles a delayed capture rejection after the presentation deadline wins", async () => { + vi.useFakeTimers(); + const runtimeTabId = addRuntimeTab("tab-background"); + vi.stubGlobal("document", { + querySelectorAll: () => [ + { + dataset: { + previewViewport: runtimeTabId, + previewBackgroundCapture: "true", + }, + offsetWidth: 800, + }, + ], + }); + vi.stubGlobal("window", { + requestAnimationFrame: (callback: FrameRequestCallback) => { + callback(0); + return 1; + }, + }); + try { + let rejectOperation!: (cause: Error) => void; + const stalledOperation = new Promise((_resolve, reject) => { + rejectOperation = reject; + }); + const operation = withPreviewAutomationBackgroundPresentation({ + threadRef, + requestId: "request-rejected-after-timeout", + tabId: "tab-background", + runtimeTabId, + timeoutMs: 40, + use: () => stalledOperation, + }); + const rejection = expect(operation).rejects.toMatchObject({ + _tag: "PreviewAutomationBackgroundPresentationTimeoutError", + requestId: "request-rejected-after-timeout", + tabId: "tab-background", + timeoutMs: 40, + }); + + await vi.advanceTimersByTimeAsync(40); + await rejection; + + rejectOperation(new Error("delayed desktop capture failure")); + await Promise.resolve(); + await Promise.resolve(); + expect( + useBrowserSurfaceStore.getState().backgroundCaptureCountByTabId[runtimeTabId], + ).toBeUndefined(); + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); }); diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index 09261b4cbdf..d0b9beedfe3 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -256,6 +256,9 @@ export async function withPreviewAutomationBackgroundPresentation( const stillBackground = !isPreviewAutomationTabPresented(input); const capture = input.use(stillBackground); captureStarted = true; + // Keep the finalized capture in the race: Promise.race retains its rejection + // handler after the deadline wins, so a delayed capture failure is observed + // while this finalizer releases the presentation lease. const operation = capture.finally(releaseCapture); const captureDeadline = new Promise((_resolve, reject) => { timer = globalThis.setTimeout(() => reject(timeoutError()), remainingMs); diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts index 517f69f92d3..05a3c8bea7e 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts @@ -20,6 +20,7 @@ import { import { createPreviewAutomationRequestConsumerAtom, previewAutomationExecutionBudget, + previewAutomationInputWithRemainingTimeout, previewAutomationRemainingBudget, serializePreviewAutomationError, } from "./previewAutomationRequestConsumer"; @@ -83,6 +84,24 @@ describe("previewAutomationRequestConsumer", () => { expect(previewAutomationRemainingBudget(1_000, 15_000, 1_001)).toBe(-1); }); + it("clamps timeout-bearing desktop inputs to the remaining host budget", () => { + const remainingOperationBudget = vi.fn((requestedTimeoutMs: number) => + Math.min(requestedTimeoutMs, 125), + ); + + expect( + previewAutomationInputWithRemainingTimeout( + { locator: "text=Continue", timeoutMs: 500 }, + 15_000, + remainingOperationBudget, + ), + ).toEqual({ locator: "text=Continue", timeoutMs: 125 }); + expect( + previewAutomationInputWithRemainingTimeout({ text: "ready" }, 750, remainingOperationBudget), + ).toEqual({ text: "ready", timeoutMs: 125 }); + expect(remainingOperationBudget.mock.calls).toEqual([[500], [750]]); + }); + it("acknowledges a replacement stream before consuming requests from it", async () => { const requestsAtom = Atom.make( AsyncResult.success({ diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts index b80a8f98a05..03d16612409 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts @@ -28,6 +28,15 @@ export const previewAutomationRemainingBudget = ( now = Date.now(), ): number => Math.min(requestedTimeoutMs, operationDeadline - now); +export const previewAutomationInputWithRemainingTimeout = ( + input: Input & { readonly timeoutMs?: number | undefined }, + requestTimeoutMs: number, + remainingOperationBudget: (requestedTimeoutMs: number) => number, +): Input & { readonly timeoutMs: number } => ({ + ...input, + timeoutMs: remainingOperationBudget(input.timeoutMs ?? requestTimeoutMs), +}); + const handleWithinResponseBudget = ( request: PreviewAutomationRequest, environmentId: PreviewAutomationHost["environmentId"], From b3f784705c3334a089e9f0eba3e66afc04893174 Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Mon, 3 Aug 2026 19:21:23 +0100 Subject: [PATCH 35/43] Clamp preview readiness waits to operation deadlines - Bound presentation settling and overlay/navigation polling - Add short-deadline regressions and refresh branch verification --- BRANCH_DETAILS.md | 11 +-- .../preview/PreviewAutomationHosts.tsx | 43 +----------- .../previewAutomationOverlayReadiness.test.ts | 68 +++++++++++++++++++ .../previewAutomationOverlayReadiness.ts | 35 ++++++++++ .../previewAutomationPresentation.test.ts | 22 ++++++ .../preview/previewAutomationPresentation.ts | 15 ++++ .../previewNavigationReadiness.test.ts | 55 ++++++++++++++- .../preview/previewNavigationReadiness.ts | 4 +- 8 files changed, 204 insertions(+), 49 deletions(-) create mode 100644 apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts create mode 100644 apps/web/src/components/preview/previewAutomationOverlayReadiness.ts diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 3568074d5fd..d66125b7d42 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -24,7 +24,7 @@ Expected behavior: - Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the server epoch replaces the runtime guest while staging is pending, the snapshot fails immediately with `PreviewAutomationTargetUnavailableError` instead of waiting on the stale staging marker. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. -- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget, and the transition into grace reservation remains monotonic as requested timeouts increase. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling, and both visibility polling and stable-presentation dwell contract to fit short deadlines instead of overshooting them or requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. +- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget, and the transition into grace reservation remains monotonic as requested timeouts increase. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling. Best-effort presentation settling clamps its 500-millisecond ceiling to the remaining operation budget, while overlay, navigation, and visibility polling clamp each sleep to the remaining deadline. Stable-presentation dwell also contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. - A newly created preview tab applies its server snapshot and assigned tab id, applies the upstream 1280×800 automation viewport when the server snapshot still uses `fill`, initiates any requested selection, and acknowledges server-side creation without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across same-server route hydration or session reconciliation instead of accepting one transient visible frame. A server-epoch change denotes a replacement runtime guest and aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the new guest. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for `open`. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - Retained browser guests subscribe only to render state for their own epoch-scoped runtime tab, plus their thread's active panel and mini-player selection. Presentation or background-capture updates for another tab do not rerender every mounted `HostedBrowserWebview`; selection changes override a stale surface-visible flag so background staging remains nearly transparent and exposes its readiness marker, and staging derives its viewport-fitted rectangle from the target tab's own stable rectangle. @@ -54,6 +54,7 @@ Primary files: - `apps/web/src/components/preview/PreviewAutomationHosts.tsx` - `apps/web/src/components/preview/previewAutomationPresentation.ts` - `apps/web/src/components/preview/previewAutomationOpenReadiness.ts` +- `apps/web/src/components/preview/previewAutomationOverlayReadiness.ts` - `apps/web/src/components/preview/previewAutomationErrors.ts` - `apps/web/src/components/preview/previewAutomationRequestConsumer.ts` - `apps/web/vite.config.ts` @@ -61,17 +62,17 @@ Primary files: - `packages/contracts/src/ipc.ts` - `scripts/dev-runner.ts` -Focused regression coverage lives in `scripts/dev-runner.test.ts`, `apps/desktop/src/app/DesktopAppIdentity.test.ts`, `apps/desktop/src/app/DesktopEnvironment.test.ts`, `apps/desktop/src/preview/Manager.test.ts`, `apps/server/src/mcp/McpHttpServer.test.ts`, `apps/web/src/browser/browserSurfaceStore.test.ts`, `apps/web/src/browser/hostedBrowserWebviewStyle.test.ts`, `apps/web/src/browser/browserViewportActions.test.ts`, `apps/web/src/browser/browserViewportLayout.test.ts`, `apps/web/src/browser/previewRuntimeTabId.test.ts`, `apps/web/src/components/auth/PairingRouteSurface.logic.test.ts`, `apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationPresentation.test.ts`, `apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts`, `apps/web/src/components/preview/previewNavigationReadiness.test.ts`, `apps/web/src/components/preview/previewViewportRollback.test.ts`, `packages/contracts/src/ipc.test.ts`, and `packages/contracts/src/preview.test.ts`. +Focused regression coverage lives in `scripts/dev-runner.test.ts`, `apps/desktop/src/app/DesktopAppIdentity.test.ts`, `apps/desktop/src/app/DesktopEnvironment.test.ts`, `apps/desktop/src/preview/Manager.test.ts`, `apps/server/src/mcp/McpHttpServer.test.ts`, `apps/web/src/browser/browserSurfaceStore.test.ts`, `apps/web/src/browser/hostedBrowserWebviewStyle.test.ts`, `apps/web/src/browser/browserViewportActions.test.ts`, `apps/web/src/browser/browserViewportLayout.test.ts`, `apps/web/src/browser/previewRuntimeTabId.test.ts`, `apps/web/src/components/auth/PairingRouteSurface.logic.test.ts`, `apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationPresentation.test.ts`, `apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts`, `apps/web/src/components/preview/previewNavigationReadiness.test.ts`, `apps/web/src/components/preview/previewViewportRollback.test.ts`, `packages/contracts/src/ipc.test.ts`, and `packages/contracts/src/preview.test.ts`. ```sh -vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.test.ts apps/desktop/src/app/DesktopEnvironment.test.ts apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/web/src/browser/browserSurfaceStore.test.ts apps/web/src/browser/hostedBrowserWebviewStyle.test.ts apps/web/src/browser/browserViewportActions.test.ts apps/web/src/browser/browserViewportLayout.test.ts apps/web/src/browser/previewRuntimeTabId.test.ts apps/web/src/components/auth/PairingRouteSurface.logic.test.ts apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts apps/web/src/components/preview/previewAutomationPresentation.test.ts apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts apps/web/src/components/preview/previewNavigationReadiness.test.ts apps/web/src/components/preview/previewViewportRollback.test.ts packages/contracts/src/ipc.test.ts packages/contracts/src/preview.test.ts +vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.test.ts apps/desktop/src/app/DesktopEnvironment.test.ts apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/web/src/browser/browserSurfaceStore.test.ts apps/web/src/browser/hostedBrowserWebviewStyle.test.ts apps/web/src/browser/browserViewportActions.test.ts apps/web/src/browser/browserViewportLayout.test.ts apps/web/src/browser/previewRuntimeTabId.test.ts apps/web/src/components/auth/PairingRouteSurface.logic.test.ts apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts apps/web/src/components/preview/previewAutomationPresentation.test.ts apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts apps/web/src/components/preview/previewNavigationReadiness.test.ts apps/web/src/components/preview/previewViewportRollback.test.ts packages/contracts/src/ipc.test.ts packages/contracts/src/preview.test.ts ``` Current verification: -- The focused command above passed all 230 tests, including remaining-host-budget propagation for timeout-bearing desktop inputs, handled delayed capture rejection after a presentation timeout, current-tab-only browser-surface selection, stale-visible background staging after selection changes, and serialized pairing submissions. +- The focused command above passed all 233 tests, including remaining-host-budget propagation for timeout-bearing desktop inputs, deadline-clamped presentation settling and overlay/navigation polling, handled delayed capture rejection after a presentation timeout, current-tab-only browser-surface selection, stale-visible background staging after selection changes, and serialized pairing submissions. - The desktop manager's 35 focused tests additionally verify that a screenshot timeout detaches the poisoned debugger session before a queued evaluation reattaches and sends its first command. -- The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 44 tests with explicit runtime guest identity, including immediate open, monotonic short-deadline handling, remaining desktop input budgets, deadline-clamped visibility polling, handled delayed capture rejection, and snapshot rejection after a server-epoch replacement. +- The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationOverlayReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 47 tests with explicit runtime guest identity, including immediate open, monotonic short-deadline handling, remaining desktop input budgets, deadline-clamped presentation, overlay, navigation, and visibility polling, handled delayed capture rejection, and snapshot rejection after a server-epoch replacement. - Desktop, server, contracts, and scripts typechecks completed without type errors. The web typecheck currently reaches only three `RegistryContext` typing errors that are also present on `upstream/main`. - An isolated worktree `dev:desktop` using the user-data override paired successfully and stayed usable through repeated renderer/CDP inspection and raster capture. An archive worktree client loaded its seeded project against that host without the prior IndexedDB `VersionError` or host-interface disappearance. - An isolated web client on ports `5744`/`13784` completed first-navigation pairing, loaded the seeded Preview Reliability thread, and rendered the right-panel surface chooser. Its non-Electron Browser surface was unavailable as described under current limitations. diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 29cd0dd9fc4..ef8b8aeb347 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -49,12 +49,12 @@ import { previewBridge } from "./previewBridge"; import { revealPreviewAutomationTab, waitForBrowserSurfaceVisibility, + waitForPreviewPresentation, withPreviewAutomationBackgroundPresentation, } from "./previewAutomationPresentation"; import { PreviewAutomationHostDeadlineExceededError, PreviewAutomationOperationError, - PreviewAutomationOverlayTimeoutError, PreviewAutomationRecordingNotActiveError, PreviewAutomationTargetUnavailableError, PreviewAutomationViewportTimeoutError, @@ -64,6 +64,7 @@ import { resolvePreviewAutomationOpenWaitPolicy, shouldOpenPreviewMiniPlayer, } from "./previewAutomationOpenReadiness"; +import { waitForDesktopOverlay } from "./previewAutomationOverlayReadiness"; import { createPreviewAutomationRequestConsumerAtom, previewAutomationExecutionBudget, @@ -83,44 +84,6 @@ import { import { isPreviewViewportReady } from "./previewViewportReadiness"; import { shouldRollbackPreviewViewport } from "./previewViewportRollback"; -const PREVIEW_PRESENTATION_SETTLE_TIMEOUT_MS = 500; - -const waitForPreviewPresentation = async (runtimeTabId: string): Promise => { - const deadline = Date.now() + PREVIEW_PRESENTATION_SETTLE_TIMEOUT_MS; - while (Date.now() <= deadline) { - if (useBrowserSurfaceStore.getState().byTabId[runtimeTabId]?.visible) return; - await new Promise((resolve) => window.setTimeout(resolve, 16)); - } -}; - -const waitForDesktopOverlay = async ( - threadRef: ScopedThreadRef, - requestId: string, - tabId: string, - runtimeTabId: string, - operation: PreviewAutomationRequest["operation"], - timeoutMs: number, -): Promise => { - const deadline = Date.now() + timeoutMs; - while (Date.now() <= deadline) { - const state = assertPreviewRuntimeCurrent(threadRef, tabId, runtimeTabId, { - operation, - requestId, - }); - if (state.desktopByTabId[tabId] && previewBridge) { - const status = await previewBridge.automation.status(runtimeTabId); - if (status.available) return; - } - await new Promise((resolve) => window.setTimeout(resolve, 50)); - } - throw new PreviewAutomationOverlayTimeoutError({ - requestId, - environmentId: threadRef.environmentId, - threadId: threadRef.threadId, - timeoutMs, - }); -}; - interface ExecutablePreviewWebview extends Element { readonly executeJavaScript: (code: string, userGesture?: boolean) => Promise; } @@ -477,7 +440,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) // briefly so active-thread opens report visible=true, without // turning a background thread's offscreen mini player into an // operation failure. - await waitForPreviewPresentation(activeRuntimeTabId); + await waitForPreviewPresentation(activeRuntimeTabId, remainingOperationBudget()); } if (reusedExistingTab && resolvedInputUrl && previewBridge) { assertPreviewRuntimeCurrent(threadRef, activeTabId, activeRuntimeTabId, request); diff --git a/apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts b/apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts new file mode 100644 index 00000000000..7ce85aeb140 --- /dev/null +++ b/apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts @@ -0,0 +1,68 @@ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { describe, expect, it, vi } from "vite-plus/test"; + +const mocks = vi.hoisted(() => ({ + readThreadPreviewState: vi.fn(), + status: vi.fn(), +})); + +vi.mock("~/previewStateStore", () => ({ + readThreadPreviewState: mocks.readThreadPreviewState, +})); + +vi.mock("./previewBridge", () => ({ + previewBridge: { + automation: { + status: mocks.status, + }, + }, +})); + +import { previewRuntimeTabId } from "~/browser/previewRuntimeTabId"; + +import { PreviewAutomationOverlayTimeoutError } from "./previewAutomationErrors"; +import { waitForDesktopOverlay } from "./previewAutomationOverlayReadiness"; + +describe("waitForDesktopOverlay", () => { + it("clamps overlay polling to a short remaining operation budget", async () => { + vi.useFakeTimers(); + vi.stubGlobal("window", { + setTimeout, + }); + const threadRef = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), + }; + const tabId = "tab-1"; + const runtimeTabId = previewRuntimeTabId(threadRef, "epoch-1", tabId); + mocks.readThreadPreviewState.mockReturnValue({ + serverEpoch: "epoch-1", + sessions: { + [tabId]: { tabId }, + }, + desktopByTabId: { + [tabId]: true, + }, + }); + mocks.status.mockResolvedValue({ available: false }); + try { + const overlay = waitForDesktopOverlay( + threadRef, + "request-1", + tabId, + runtimeTabId, + "open", + 40, + ); + const rejection = expect(overlay).rejects.toBeInstanceOf( + PreviewAutomationOverlayTimeoutError, + ); + + await vi.advanceTimersByTimeAsync(40); + await rejection; + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); +}); diff --git a/apps/web/src/components/preview/previewAutomationOverlayReadiness.ts b/apps/web/src/components/preview/previewAutomationOverlayReadiness.ts new file mode 100644 index 00000000000..9aeffbb88e8 --- /dev/null +++ b/apps/web/src/components/preview/previewAutomationOverlayReadiness.ts @@ -0,0 +1,35 @@ +import { type PreviewAutomationRequest, type ScopedThreadRef } from "@t3tools/contracts"; + +import { previewBridge } from "./previewBridge"; +import { PreviewAutomationOverlayTimeoutError } from "./previewAutomationErrors"; +import { assertPreviewRuntimeCurrent } from "./previewNavigationReadiness"; + +export async function waitForDesktopOverlay( + threadRef: ScopedThreadRef, + requestId: string, + tabId: string, + runtimeTabId: string, + operation: PreviewAutomationRequest["operation"], + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() <= deadline) { + const state = assertPreviewRuntimeCurrent(threadRef, tabId, runtimeTabId, { + operation, + requestId, + }); + if (state.desktopByTabId[tabId] && previewBridge) { + const status = await previewBridge.automation.status(runtimeTabId); + if (status.available) return; + } + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) break; + await new Promise((resolve) => window.setTimeout(resolve, Math.min(50, remainingMs))); + } + throw new PreviewAutomationOverlayTimeoutError({ + requestId, + environmentId: threadRef.environmentId, + threadId: threadRef.threadId, + timeoutMs, + }); +} diff --git a/apps/web/src/components/preview/previewAutomationPresentation.test.ts b/apps/web/src/components/preview/previewAutomationPresentation.test.ts index 726c5e0c1ba..c34c0941567 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.test.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.test.ts @@ -18,6 +18,7 @@ import { readPreviewAutomationPresentationDiagnostics, revealPreviewAutomationTab, waitForBrowserSurfaceVisibility, + waitForPreviewPresentation, withPreviewAutomationBackgroundPresentation, waitForPreviewAutomationBackgroundPresentation, } from "./previewAutomationPresentation"; @@ -264,6 +265,27 @@ describe("preview automation presentation", () => { } }); + it("clamps best-effort presentation settling to the remaining operation budget", async () => { + vi.useFakeTimers(); + vi.stubGlobal("window", { + setTimeout, + }); + try { + const settled = vi.fn(); + const presentation = waitForPreviewPresentation("tab-open", 40).then(settled); + + await vi.advanceTimersByTimeAsync(39); + expect(settled).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(1); + await presentation; + expect(settled).toHaveBeenCalledTimes(1); + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); + it("rejects an open visibility wait when the runtime guest is replaced", async () => { vi.useFakeTimers(); vi.stubGlobal("window", { diff --git a/apps/web/src/components/preview/previewAutomationPresentation.ts b/apps/web/src/components/preview/previewAutomationPresentation.ts index d0b9beedfe3..ed3809909d8 100644 --- a/apps/web/src/components/preview/previewAutomationPresentation.ts +++ b/apps/web/src/components/preview/previewAutomationPresentation.ts @@ -146,6 +146,21 @@ export function isPreviewAutomationTabPresented( return requestedSurfaceIsActive && (presentation?.visible ?? false); } +const PREVIEW_PRESENTATION_SETTLE_TIMEOUT_MS = 500; + +export async function waitForPreviewPresentation( + runtimeTabId: string, + timeoutMs: number, +): Promise { + const deadline = Date.now() + Math.min(PREVIEW_PRESENTATION_SETTLE_TIMEOUT_MS, timeoutMs); + while (true) { + if (useBrowserSurfaceStore.getState().byTabId[runtimeTabId]?.visible) return; + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) return; + await new Promise((resolve) => window.setTimeout(resolve, Math.min(16, remainingMs))); + } +} + export async function waitForBrowserSurfaceVisibility( input: PreviewAutomationVisibilityInput, ): Promise { diff --git a/apps/web/src/components/preview/previewNavigationReadiness.test.ts b/apps/web/src/components/preview/previewNavigationReadiness.test.ts index cefff718413..1f127e99970 100644 --- a/apps/web/src/components/preview/previewNavigationReadiness.test.ts +++ b/apps/web/src/components/preview/previewNavigationReadiness.test.ts @@ -1,8 +1,9 @@ import { EnvironmentId, ThreadId } from "@t3tools/contracts"; -import { describe, expect, it, vi } from "vite-plus/test"; +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; const mocks = vi.hoisted(() => ({ readThreadPreviewState: vi.fn(), + status: vi.fn(), })); vi.mock("~/previewStateStore", () => ({ @@ -16,17 +17,25 @@ vi.mock("./previewBridge", () => ({ previewBridge: { automation: { evaluate: vi.fn(), - status: vi.fn(), + status: mocks.status, }, }, })); import { previewRuntimeTabId } from "~/browser/previewRuntimeTabId"; -import { PreviewAutomationTargetUnavailableError } from "./previewAutomationErrors"; +import { + PreviewAutomationNavigationTimeoutError, + PreviewAutomationTargetUnavailableError, +} from "./previewAutomationErrors"; import { waitForNavigationReadiness } from "./previewNavigationReadiness"; describe("waitForNavigationReadiness", () => { + beforeEach(() => { + mocks.readThreadPreviewState.mockReset(); + mocks.status.mockReset(); + }); + it("rejects a replaced runtime target even when readiness polling is disabled", async () => { const threadRef = { environmentId: EnvironmentId.make("environment-2"), @@ -53,4 +62,44 @@ describe("waitForNavigationReadiness", () => { ), ).rejects.toBeInstanceOf(PreviewAutomationTargetUnavailableError); }); + + it("clamps readiness polling to a short remaining operation budget", async () => { + vi.useFakeTimers(); + vi.stubGlobal("window", { + setTimeout, + }); + const threadRef = { + environmentId: EnvironmentId.make("environment-2"), + threadId: ThreadId.make("thread-1"), + }; + const tabId = "tab_1"; + const runtimeTabId = previewRuntimeTabId(threadRef, "epoch-1", tabId); + mocks.readThreadPreviewState.mockReturnValue({ + serverEpoch: "epoch-1", + sessions: { + [tabId]: { tabId }, + }, + }); + mocks.status.mockResolvedValue({ available: true, loading: true }); + try { + const readiness = waitForNavigationReadiness( + threadRef, + "request-1", + tabId, + runtimeTabId, + "navigate", + "load", + 40, + ); + const rejection = expect(readiness).rejects.toBeInstanceOf( + PreviewAutomationNavigationTimeoutError, + ); + + await vi.advanceTimersByTimeAsync(40); + await rejection; + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); }); diff --git a/apps/web/src/components/preview/previewNavigationReadiness.ts b/apps/web/src/components/preview/previewNavigationReadiness.ts index dd5e7247b71..ff9d00b3b6e 100644 --- a/apps/web/src/components/preview/previewNavigationReadiness.ts +++ b/apps/web/src/components/preview/previewNavigationReadiness.ts @@ -61,7 +61,9 @@ export async function waitForNavigationReadiness( const status = await previewBridge.automation.status(runtimeTabId); if (status.available && !status.loading) return; } - await new Promise((resolve) => window.setTimeout(resolve, 50)); + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) break; + await new Promise((resolve) => window.setTimeout(resolve, Math.min(50, remainingMs))); } throw new PreviewAutomationNavigationTimeoutError({ requestId, From 12776c7824f95794aaec7fe679b880b4208d798b Mon Sep 17 00:00:00 2001 From: T3 Verification Date: Mon, 3 Aug 2026 19:37:20 +0100 Subject: [PATCH 36/43] Bound preview overlay readiness status calls - Bound status IPC and revalidate the runtime guest - Keep best-effort settle budget reads non-throwing --- BRANCH_DETAILS.md | 6 +- .../preview/PreviewAutomationHosts.tsx | 6 +- .../previewAutomationOverlayReadiness.test.ts | 102 +++++++++++++++++- .../previewAutomationOverlayReadiness.ts | 30 ++++-- .../previewAutomationRequestConsumer.test.ts | 6 ++ .../previewAutomationRequestConsumer.ts | 7 ++ 6 files changed, 145 insertions(+), 12 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index d66125b7d42..9c927787375 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -24,7 +24,7 @@ Expected behavior: - Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the server epoch replaces the runtime guest while staging is pending, the snapshot fails immediately with `PreviewAutomationTargetUnavailableError` instead of waiting on the stale staging marker. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. -- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget, and the transition into grace reservation remains monotonic as requested timeouts increase. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling. Best-effort presentation settling clamps its 500-millisecond ceiling to the remaining operation budget, while overlay, navigation, and visibility polling clamp each sleep to the remaining deadline. Stable-presentation dwell also contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. +- The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget, and the transition into grace reservation remains monotonic as requested timeouts increase. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling. Best-effort presentation settling uses a non-throwing remaining-budget read and clamps its 500-millisecond ceiling to that budget. Overlay status calls are themselves bounded by the remaining deadline and revalidate runtime guest identity after awaiting the desktop bridge, while overlay, navigation, and visibility polling clamp each sleep to the remaining deadline. Stable-presentation dwell also contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. - A newly created preview tab applies its server snapshot and assigned tab id, applies the upstream 1280×800 automation viewport when the server snapshot still uses `fill`, initiates any requested selection, and acknowledges server-side creation without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across same-server route hydration or session reconciliation instead of accepting one transient visible frame. A server-epoch change denotes a replacement runtime guest and aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the new guest. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for `open`. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - Retained browser guests subscribe only to render state for their own epoch-scoped runtime tab, plus their thread's active panel and mini-player selection. Presentation or background-capture updates for another tab do not rerender every mounted `HostedBrowserWebview`; selection changes override a stale surface-visible flag so background staging remains nearly transparent and exposes its readiness marker, and staging derives its viewport-fitted rectangle from the target tab's own stable rectangle. @@ -70,9 +70,9 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.t Current verification: -- The focused command above passed all 233 tests, including remaining-host-budget propagation for timeout-bearing desktop inputs, deadline-clamped presentation settling and overlay/navigation polling, handled delayed capture rejection after a presentation timeout, current-tab-only browser-surface selection, stale-visible background staging after selection changes, and serialized pairing submissions. +- The focused command above passed all 236 tests, including remaining-host-budget propagation for timeout-bearing desktop inputs, non-throwing best-effort presentation settling, deadline-bounded overlay status and polling, post-status runtime identity validation, deadline-clamped navigation polling, handled delayed capture rejection after a presentation timeout, current-tab-only browser-surface selection, stale-visible background staging after selection changes, and serialized pairing submissions. - The desktop manager's 35 focused tests additionally verify that a screenshot timeout detaches the poisoned debugger session before a queued evaluation reattaches and sends its first command. -- The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationOverlayReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 47 tests with explicit runtime guest identity, including immediate open, monotonic short-deadline handling, remaining desktop input budgets, deadline-clamped presentation, overlay, navigation, and visibility polling, handled delayed capture rejection, and snapshot rejection after a server-epoch replacement. +- The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationOverlayReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 50 tests with explicit runtime guest identity, including immediate open, monotonic short-deadline handling, remaining desktop input budgets, non-throwing best-effort settling, bounded overlay status calls, post-status runtime replacement rejection, deadline-clamped presentation, overlay, navigation, and visibility polling, handled delayed capture rejection, and snapshot rejection after a server-epoch replacement. - Desktop, server, contracts, and scripts typechecks completed without type errors. The web typecheck currently reaches only three `RegistryContext` typing errors that are also present on `upstream/main`. - An isolated worktree `dev:desktop` using the user-data override paired successfully and stayed usable through repeated renderer/CDP inspection and raster capture. An archive worktree client loaded its seeded project against that host without the prior IndexedDB `VersionError` or host-interface disappearance. - An isolated web client on ports `5744`/`13784` completed first-navigation pairing, loaded the seeded Preview Reliability thread, and rendered the right-panel surface chooser. Its non-Electron Browser surface was unavailable as described under current limitations. diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index ef8b8aeb347..40ba113ac8d 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -69,6 +69,7 @@ import { createPreviewAutomationRequestConsumerAtom, previewAutomationExecutionBudget, previewAutomationInputWithRemainingTimeout, + previewAutomationRemainingBestEffortBudget, previewAutomationRemainingBudget, } from "./previewAutomationRequestConsumer"; import { @@ -440,7 +441,10 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) // briefly so active-thread opens report visible=true, without // turning a background thread's offscreen mini player into an // operation failure. - await waitForPreviewPresentation(activeRuntimeTabId, remainingOperationBudget()); + await waitForPreviewPresentation( + activeRuntimeTabId, + previewAutomationRemainingBestEffortBudget(operationDeadline, request.timeoutMs), + ); } if (reusedExistingTab && resolvedInputUrl && previewBridge) { assertPreviewRuntimeCurrent(threadRef, activeTabId, activeRuntimeTabId, request); diff --git a/apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts b/apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts index 7ce85aeb140..9f56431333a 100644 --- a/apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts +++ b/apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts @@ -20,13 +20,17 @@ vi.mock("./previewBridge", () => ({ import { previewRuntimeTabId } from "~/browser/previewRuntimeTabId"; -import { PreviewAutomationOverlayTimeoutError } from "./previewAutomationErrors"; +import { + PreviewAutomationOverlayTimeoutError, + PreviewAutomationTargetUnavailableError, +} from "./previewAutomationErrors"; import { waitForDesktopOverlay } from "./previewAutomationOverlayReadiness"; describe("waitForDesktopOverlay", () => { it("clamps overlay polling to a short remaining operation budget", async () => { vi.useFakeTimers(); vi.stubGlobal("window", { + clearTimeout, setTimeout, }); const threadRef = { @@ -65,4 +69,100 @@ describe("waitForDesktopOverlay", () => { vi.useRealTimers(); } }); + + it("bounds a stalled overlay status call by the remaining deadline", async () => { + vi.useFakeTimers(); + vi.stubGlobal("window", { + clearTimeout, + setTimeout, + }); + const threadRef = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), + }; + const tabId = "tab-1"; + const runtimeTabId = previewRuntimeTabId(threadRef, "epoch-1", tabId); + mocks.readThreadPreviewState.mockReturnValue({ + serverEpoch: "epoch-1", + sessions: { + [tabId]: { tabId }, + }, + desktopByTabId: { + [tabId]: true, + }, + }); + mocks.status.mockReturnValue(new Promise(() => undefined)); + try { + const overlay = waitForDesktopOverlay( + threadRef, + "request-1", + tabId, + runtimeTabId, + "open", + 40, + ); + const rejection = expect(overlay).rejects.toBeInstanceOf( + PreviewAutomationOverlayTimeoutError, + ); + + await vi.advanceTimersByTimeAsync(40); + await rejection; + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); + + it("rejects a runtime guest replaced while overlay status is pending", async () => { + vi.stubGlobal("window", { + clearTimeout, + setTimeout, + }); + const threadRef = { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), + }; + const tabId = "tab-1"; + const runtimeTabId = previewRuntimeTabId(threadRef, "epoch-1", tabId); + mocks.readThreadPreviewState.mockReturnValue({ + serverEpoch: "epoch-1", + sessions: { + [tabId]: { tabId }, + }, + desktopByTabId: { + [tabId]: true, + }, + }); + let resolveStatus!: (status: { readonly available: boolean }) => void; + mocks.status.mockReturnValue( + new Promise((resolve) => { + resolveStatus = resolve; + }), + ); + try { + const overlay = waitForDesktopOverlay( + threadRef, + "request-1", + tabId, + runtimeTabId, + "open", + 1_000, + ); + + mocks.readThreadPreviewState.mockReturnValue({ + serverEpoch: "epoch-2", + sessions: { + [tabId]: { tabId }, + }, + desktopByTabId: { + [tabId]: true, + }, + }); + resolveStatus({ available: true }); + + await expect(overlay).rejects.toBeInstanceOf(PreviewAutomationTargetUnavailableError); + } finally { + vi.unstubAllGlobals(); + } + }); }); diff --git a/apps/web/src/components/preview/previewAutomationOverlayReadiness.ts b/apps/web/src/components/preview/previewAutomationOverlayReadiness.ts index 9aeffbb88e8..e3aef36d75c 100644 --- a/apps/web/src/components/preview/previewAutomationOverlayReadiness.ts +++ b/apps/web/src/components/preview/previewAutomationOverlayReadiness.ts @@ -13,23 +13,39 @@ export async function waitForDesktopOverlay( timeoutMs: number, ): Promise { const deadline = Date.now() + timeoutMs; + const timeoutError = () => + new PreviewAutomationOverlayTimeoutError({ + requestId, + environmentId: threadRef.environmentId, + threadId: threadRef.threadId, + timeoutMs, + }); while (Date.now() <= deadline) { const state = assertPreviewRuntimeCurrent(threadRef, tabId, runtimeTabId, { operation, requestId, }); if (state.desktopByTabId[tabId] && previewBridge) { - const status = await previewBridge.automation.status(runtimeTabId); + const remainingStatusBudgetMs = deadline - Date.now(); + if (remainingStatusBudgetMs <= 0) break; + let timeoutId: number | undefined; + const status = await Promise.race([ + previewBridge.automation.status(runtimeTabId), + new Promise((_resolve, reject) => { + timeoutId = window.setTimeout(() => reject(timeoutError()), remainingStatusBudgetMs); + }), + ]).finally(() => { + if (timeoutId !== undefined) window.clearTimeout(timeoutId); + }); + assertPreviewRuntimeCurrent(threadRef, tabId, runtimeTabId, { + operation, + requestId, + }); if (status.available) return; } const remainingMs = deadline - Date.now(); if (remainingMs <= 0) break; await new Promise((resolve) => window.setTimeout(resolve, Math.min(50, remainingMs))); } - throw new PreviewAutomationOverlayTimeoutError({ - requestId, - environmentId: threadRef.environmentId, - threadId: threadRef.threadId, - timeoutMs, - }); + throw timeoutError(); } diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts index 05a3c8bea7e..8b09c2d6934 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts @@ -21,6 +21,7 @@ import { createPreviewAutomationRequestConsumerAtom, previewAutomationExecutionBudget, previewAutomationInputWithRemainingTimeout, + previewAutomationRemainingBestEffortBudget, previewAutomationRemainingBudget, serializePreviewAutomationError, } from "./previewAutomationRequestConsumer"; @@ -84,6 +85,11 @@ describe("previewAutomationRequestConsumer", () => { expect(previewAutomationRemainingBudget(1_000, 15_000, 1_001)).toBe(-1); }); + it("clamps an expired best-effort operation budget to zero", () => { + expect(previewAutomationRemainingBestEffortBudget(1_000, 15_000, 999)).toBe(1); + expect(previewAutomationRemainingBestEffortBudget(1_000, 15_000, 1_001)).toBe(0); + }); + it("clamps timeout-bearing desktop inputs to the remaining host budget", () => { const remainingOperationBudget = vi.fn((requestedTimeoutMs: number) => Math.min(requestedTimeoutMs, 125), diff --git a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts index 03d16612409..bf8afc551af 100644 --- a/apps/web/src/components/preview/previewAutomationRequestConsumer.ts +++ b/apps/web/src/components/preview/previewAutomationRequestConsumer.ts @@ -28,6 +28,13 @@ export const previewAutomationRemainingBudget = ( now = Date.now(), ): number => Math.min(requestedTimeoutMs, operationDeadline - now); +export const previewAutomationRemainingBestEffortBudget = ( + operationDeadline: number, + requestedTimeoutMs: number, + now = Date.now(), +): number => + Math.max(0, previewAutomationRemainingBudget(operationDeadline, requestedTimeoutMs, now)); + export const previewAutomationInputWithRemainingTimeout = ( input: Input & { readonly timeoutMs?: number | undefined }, requestTimeoutMs: number, From 8cd9ac3f3951c175d5f072ea48293e55addee462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Mon, 17 Aug 2026 11:29:41 +0100 Subject: [PATCH 37/43] fix(web): keep preview open decisions consistent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - resolve browser defaults once per automation open request - pass one presentation decision into reused-tab readiness - document merged defaults and focused verification results 🤖 Co-authored by GPT-5 in Codex via T3 Code --- BRANCH_DETAILS.md | 22 ++++++------ .../preview/PreviewAutomationHosts.tsx | 15 ++++---- .../previewAutomationOpenReadiness.test.ts | 36 ++++++++++++++++++- .../preview/previewAutomationOpenReadiness.ts | 5 ++- 4 files changed, 56 insertions(+), 22 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 9c927787375..9c592a615b5 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -13,6 +13,8 @@ Upstream commit `f4c394323` (`Add background preview capture and picture-in-pict Upstream commit `32af2f002` (`fix(preview): stabilize PiP viewport identity`) owns epoch-scoped runtime guest identity and keeps PiP, recording, renderer surfaces, and Electron tabs aligned on that identity. +Upstream commit `949feb61` (`feat(web): configurable browser defaults`) owns the persisted viewport, zoom, appearance, and automatic floating-preview defaults, plus applying those defaults when a browser tab is created. Upstream commit `cd096b9ad` (`feat(server): users can withhold browser access from agents`) owns whether preview tools and instructions are exposed to a provider session. The branch composes with those settings; it does not maintain another browser-default or access-control layer. + This branch does not add a second recording/PiP capture lifecycle or another hidden-preview lifetime mechanism. Its remaining background-specific code is limited to bounded, one-shot automation snapshots. That distinction is necessary: a fresh offscreen guest did not produce a frame or settle upstream recording startup within eight seconds, while staging the same guest at effectively transparent opacity produced a complete screenshot and semantic snapshot in under 100 milliseconds. Upstream's retrying frame loop therefore does not replace the automation snapshot presentation lease, exact-target CDP capture, request deadline propagation, or nullable semantic fallback. Expected behavior: @@ -20,12 +22,12 @@ Expected behavior: - Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout without making the execution budget shrink when the caller increases a short timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session while still holding an acquired control permit when a CDP command may be pending. Session removal and debugger teardown are atomic with respect to new session acquisition and bound to the exact acquired session, so late interruption or snapshot cleanup cannot detach a healthy replacement. Operations already queued on the retired semaphore detect that stale session and retry against its replacement. A request that times out while queued behind another action does not detach that action's shared debugger session. - Click, type, and wait operations clamp their caller-supplied timeout to the remaining renderer host budget before entering the desktop control-session boundary. Operations without a caller timeout use the remaining bounded request budget rather than restarting the desktop default after renderer readiness work. - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Primary and fallback screenshot waits are clamped to the remaining control-session deadline, with budget reserved for fallback and result settlement, so a tight caller deadline can still return semantic data instead of being preempted by the outer session timeout. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, an actually timed-out CDP capture resets the session before releasing its control permit, queued work reattaches before issuing its first command, and a capture skipped before CDP runs leaves the healthy session attached. The semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. -- Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. A persisted non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path; tabs following the system scheme stay detached until the next automation operation. +- Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. `apps/web/src/browser/desktopTabLifetime.ts` passes the upstream browser appearance default through `DesktopPreviewCreateTabInputSchema` in `packages/contracts/src/ipc.ts`; `apps/desktop/src/preview/Manager.ts` normalizes that value. A non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path, while tabs following the system scheme stay detached until the next automation operation. - Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the server epoch replaces the runtime guest while staging is pending, the snapshot fails immediately with `PreviewAutomationTargetUnavailableError` instead of waiting on the stale staging marker. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. -- The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. The desktop snapshot IPC schema and preload adapter default an omitted `background` flag to `false`, preserving foreground-capture behavior for legacy callers. +- The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. In `packages/contracts/src/ipc.ts`, the branch's desktop snapshot schema and upstream's create-tab defaults coexist: snapshot calls default an omitted `background` flag to `false`, while tab creation carries viewport, zoom, and color-scheme defaults through the preload and desktop manager. - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget, and the transition into grace reservation remains monotonic as requested timeouts increase. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling. Best-effort presentation settling uses a non-throwing remaining-budget read and clamps its 500-millisecond ceiling to that budget. Overlay status calls are themselves bounded by the remaining deadline and revalidate runtime guest identity after awaiting the desktop bridge, while overlay, navigation, and visibility polling clamp each sleep to the remaining deadline. Stable-presentation dwell also contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. -- A newly created preview tab applies its server snapshot and assigned tab id, applies the upstream 1280×800 automation viewport when the server snapshot still uses `fill`, initiates any requested selection, and acknowledges server-side creation without making the first call depend on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; status can report progress, while later wait, snapshot, or interaction operations own any attachment or page-readiness wait. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation; while the request remains pending it reasserts that explicit selection across same-server route hydration or session reconciliation instead of accepting one transient visible frame. A server-epoch change denotes a replacement runtime guest and aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the new guest. Reused tabs retain overlay, navigation, and requested-visibility readiness checks because their existing automation target should already be available. The deprecated `show` input remains an alias for `open`. +- `PreviewAutomationHosts.tsx` resolves `browserDefaults.ts` once at the start of each automation-open request. That single snapshot supplies the new-tab viewport and the automatic floating-preview preference, so creation and presentation cannot observe different settings during one request. Explicit `open` or its deprecated `show` alias remains authoritative; when both are omitted, `autoShowFloatingPreview` decides presentation. The resulting `shouldPresentPreview` value is passed unchanged to `previewAutomationOpenReadiness.ts`, so a reused rendered tab left in the background does not wait for visibility while an explicitly shown tab does. A newly created tab applies its server snapshot and assigned tab id, uses the configured viewport or the branch's deterministic 1280×800 fallback when the snapshot remains `fill`, initiates any requested selection, and acknowledges server-side creation without depending on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; later wait, snapshot, or interaction operations own attachment and page readiness. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation and reasserts that selection across same-server route hydration or session reconciliation. A server-epoch change aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the replacement runtime guest. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - Retained browser guests subscribe only to render state for their own epoch-scoped runtime tab, plus their thread's active panel and mini-player selection. Presentation or background-capture updates for another tab do not rerender every mounted `HostedBrowserWebview`; selection changes override a stale surface-visible flag so background staging remains nearly transparent and exposes its readiness marker, and staging derives its viewport-fitted rectangle from the target tab's own stable rectangle. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. Multiple tokens received while an exchange is pending are serialized, and the submitting state remains active until every queued exchange settles. @@ -47,7 +49,9 @@ Primary files: - `apps/server/src/mcp/McpHttpServer.ts` - `apps/server/src/mcp/toolkits/preview/tools.ts` - `apps/web/src/browser/HostedBrowserWebview.tsx` +- `apps/web/src/browser/browserDefaults.ts` - `apps/web/src/browser/browserSurfaceStore.ts` +- `apps/web/src/browser/desktopTabLifetime.ts` - `apps/web/src/browser/hostedBrowserWebviewStyle.ts` - `apps/web/src/components/auth/PairingRouteSurface.logic.ts` - `apps/web/src/components/auth/PairingRouteSurface.tsx` @@ -70,13 +74,11 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.t Current verification: -- The focused command above passed all 236 tests, including remaining-host-budget propagation for timeout-bearing desktop inputs, non-throwing best-effort presentation settling, deadline-bounded overlay status and polling, post-status runtime identity validation, deadline-clamped navigation polling, handled delayed capture rejection after a presentation timeout, current-tab-only browser-surface selection, stale-visible background staging after selection changes, and serialized pairing submissions. -- The desktop manager's 35 focused tests additionally verify that a screenshot timeout detaches the poisoned debugger session before a queued evaluation reattaches and sends its first command. -- The runtime-id presentation subset (`previewAutomationPresentation`, `previewAutomationOpenReadiness`, `previewAutomationOverlayReadiness`, `previewAutomationRequestConsumer`, and `previewNavigationReadiness`) passed all 50 tests with explicit runtime guest identity, including immediate open, monotonic short-deadline handling, remaining desktop input budgets, non-throwing best-effort settling, bounded overlay status calls, post-status runtime replacement rejection, deadline-clamped presentation, overlay, navigation, and visibility polling, handled delayed capture rejection, and snapshot rejection after a server-epoch replacement. -- Desktop, server, contracts, and scripts typechecks completed without type errors. The web typecheck currently reaches only three `RegistryContext` typing errors that are also present on `upstream/main`. -- An isolated worktree `dev:desktop` using the user-data override paired successfully and stayed usable through repeated renderer/CDP inspection and raster capture. An archive worktree client loaded its seeded project against that host without the prior IndexedDB `VersionError` or host-interface disappearance. -- An isolated web client on ports `5744`/`13784` completed first-navigation pairing, loaded the seeded Preview Reliability thread, and rendered the right-panel surface chooser. Its non-Electron Browser surface was unavailable as described under current limitations. -- A second isolated web pass on ports `5744`/`13784` delayed the first browser-session exchange, injected another fragment token while it was pending, and verified from in-page request timestamps that the second exchange started only after the first finished; the client then loaded the authenticated app. +- The branch-focused suite passed 254 tests across 17 files on Windows. The two unchanged desktop path-fixture files in the command above currently produce eight POSIX-versus-Windows path assertion failures; the affected runtime behavior passes in the remaining focused suite. +- The incoming browser-default, provider-access, pull-request budget, tooltip, update-copy, and settings coverage passed 552 tests across 26 files. The focused open-policy and open-session subset passes 18 tests, including a shared explicit-over-default presentation decision and a reused rendered tab that remains in the background without a visibility wait. +- Desktop, server, contracts, and mobile typechecks pass. Web typecheck currently reports 16 existing errors in `BranchToolbarBranchSelector.tsx`, `ModelPickerContent.tsx`, `PreviewAutomationHosts.tsx` at the pre-existing registry access, `FontFamilyPicker.tsx`, `use-atom-command.ts`, and `use-atom-query-runner.ts`; none are in the browser-default integration changed here. +- An isolated web client on ports `5744`/`13784` paired and loaded successfully. Settings → Integrations exposed the browser defaults and agent-access setting; agent browser access could be disabled and restored. The non-Electron client correctly disabled desktop-only viewport, zoom, and appearance controls. +- Android paired with an isolated server, loaded the seeded project, created a thread, and received a response. The changed Android thread-settings header remains visually unverified because Metro first exhausted file handles and a clean restart then exposed an installed-dev-client `EventEmitter` runtime mismatch. ## Development Ports diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 8fd72b18817..9e6991f147a 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -347,6 +347,11 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) return await currentStatus(threadRef, tabId); case "open": { const input = request.input as PreviewAutomationOpenInput; + const browserDefaults = await resolveBrowserDefaults(); + const shouldPresentPreview = shouldOpenPreviewMiniPlayer( + input, + browserDefaults.autoShowFloatingPreview, + ); const resolvedInputUrl = input.url ? resolveBrowserNavigationTarget(environmentId, { kind: "url", @@ -371,7 +376,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) ...(resolvedInputUrl ? { url: resolvedInputUrl } : {}), // An agent that didn't state a size gets the user's // configured default, same as a hand-opened tab. - viewport: browserDefaultOpenViewport(await resolveBrowserDefaults()), + viewport: browserDefaultOpenViewport(browserDefaults), }, }); if (result._tag === "Failure") { @@ -420,12 +425,6 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) updatePreviewServerSnapshot(threadRef, resizeResult.value); } } - const autoShowFloatingPreview = (await resolveBrowserDefaults()) - .autoShowFloatingPreview; - const shouldPresentPreview = shouldOpenPreviewMiniPlayer( - input, - autoShowFloatingPreview, - ); if (shouldPresentPreview) { revealPreviewAutomationTab(threadRef, activeTabId); } @@ -434,7 +433,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) input, activeSnapshot, reusedExistingTab, - autoShowFloatingPreview, + shouldPresentPreview, ) : null; if (waitPolicy?.acknowledgeAfterCreation) { diff --git a/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts b/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts index 074b95a2a01..c278661b10a 100644 --- a/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts +++ b/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts @@ -50,6 +50,7 @@ describe("preview automation open readiness", () => { title: "", }), false, + true, ), ).toEqual({ acknowledgeAfterCreation: true, @@ -68,6 +69,7 @@ describe("preview automation open readiness", () => { title: "", }), false, + false, ), ).toEqual({ acknowledgeAfterCreation: true, @@ -82,6 +84,7 @@ describe("preview automation open readiness", () => { { url: "https://example.com" } as PreviewAutomationOpenInput, snapshot({ _tag: "Idle" }), true, + true, ), ).toEqual({ acknowledgeAfterCreation: false, @@ -100,6 +103,7 @@ describe("preview automation open readiness", () => { title: "Example", }), true, + false, ), ).toEqual({ acknowledgeAfterCreation: false, @@ -114,6 +118,7 @@ describe("preview automation open readiness", () => { {} as PreviewAutomationOpenInput, snapshot({ _tag: "Idle" }), true, + true, ), ).toEqual({ acknowledgeAfterCreation: false, @@ -134,6 +139,7 @@ describe("preview automation open readiness", () => { description: "Failed", }), true, + true, ), ).toEqual({ acknowledgeAfterCreation: false, @@ -152,7 +158,7 @@ describe("preview automation open readiness", () => { title: "Example", }), true, - false, + shouldOpenPreviewMiniPlayer({}, false), ), ).toEqual({ acknowledgeAfterCreation: false, @@ -191,4 +197,32 @@ describe("shouldOpenPreviewMiniPlayer with the floating-preview preference", () expect(shouldOpenPreviewMiniPlayer({ open: false }, true)).toBe(false); expect(shouldOpenPreviewMiniPlayer({ show: true }, false)).toBe(true); }); + + it("shares the resolved explicit-over-default policy with reused-tab readiness", () => { + const renderedSnapshot = snapshot({ + _tag: "Success", + url: "https://example.com/", + title: "Example", + }); + + const explicitShow = { open: true } as PreviewAutomationOpenInput; + expect( + resolvePreviewAutomationOpenWaitPolicy( + explicitShow, + renderedSnapshot, + true, + shouldOpenPreviewMiniPlayer(explicitShow, false), + ).waitForVisibility, + ).toBe(true); + + const explicitBackground = { open: false } as PreviewAutomationOpenInput; + expect( + resolvePreviewAutomationOpenWaitPolicy( + explicitBackground, + renderedSnapshot, + true, + shouldOpenPreviewMiniPlayer(explicitBackground, true), + ).waitForVisibility, + ).toBe(false); + }); }); diff --git a/apps/web/src/components/preview/previewAutomationOpenReadiness.ts b/apps/web/src/components/preview/previewAutomationOpenReadiness.ts index 1e4cda3fc1f..0b8e7b918b9 100644 --- a/apps/web/src/components/preview/previewAutomationOpenReadiness.ts +++ b/apps/web/src/components/preview/previewAutomationOpenReadiness.ts @@ -46,7 +46,7 @@ export function resolvePreviewAutomationOpenWaitPolicy( input: PreviewAutomationOpenInput, snapshot: PreviewSessionSnapshot, reusedExistingTab: boolean, - autoShowFloatingPreview = true, + shouldPresentPreview: boolean, ): PreviewAutomationOpenWaitPolicy { if (!reusedExistingTab) { return { @@ -62,8 +62,7 @@ export function resolvePreviewAutomationOpenWaitPolicy( return { acknowledgeAfterCreation: false, waitForOverlay: previewAutomationOpenNeedsOverlay(input, snapshot), - waitForVisibility: - shouldOpenPreviewMiniPlayer(input, autoShowFloatingPreview) && canPresentBrowserSurface, + waitForVisibility: shouldPresentPreview && canPresentBrowserSurface, }; } From c649f2d3bc54d64c46792afa3171a99f41019e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Mon, 17 Aug 2026 11:43:42 +0100 Subject: [PATCH 38/43] fix(web): pin preview state after settings hydration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - hydrate browser defaults before reading the open request session - prevent reused-tab opens from mixing server epochs 🤖 Co-authored by GPT-5 in Codex via T3 Code --- BRANCH_DETAILS.md | 2 +- .../src/components/preview/PreviewAutomationHosts.tsx | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 9c592a615b5..0d38309ba58 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -27,7 +27,7 @@ Expected behavior: - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. In `packages/contracts/src/ipc.ts`, the branch's desktop snapshot schema and upstream's create-tab defaults coexist: snapshot calls default an omitted `background` flag to `false`, while tab creation carries viewport, zoom, and color-scheme defaults through the preload and desktop manager. - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget, and the transition into grace reservation remains monotonic as requested timeouts increase. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling. Best-effort presentation settling uses a non-throwing remaining-budget read and clamps its 500-millisecond ceiling to that budget. Overlay status calls are themselves bounded by the remaining deadline and revalidate runtime guest identity after awaiting the desktop bridge, while overlay, navigation, and visibility polling clamp each sleep to the remaining deadline. Stable-presentation dwell also contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. -- `PreviewAutomationHosts.tsx` resolves `browserDefaults.ts` once at the start of each automation-open request. That single snapshot supplies the new-tab viewport and the automatic floating-preview preference, so creation and presentation cannot observe different settings during one request. Explicit `open` or its deprecated `show` alias remains authoritative; when both are omitted, `autoShowFloatingPreview` decides presentation. The resulting `shouldPresentPreview` value is passed unchanged to `previewAutomationOpenReadiness.ts`, so a reused rendered tab left in the background does not wait for visibility while an explicitly shown tab does. A newly created tab applies its server snapshot and assigned tab id, uses the configured viewport or the branch's deterministic 1280×800 fallback when the snapshot remains `fill`, initiates any requested selection, and acknowledges server-side creation without depending on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; later wait, snapshot, or interaction operations own attachment and page readiness. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation and reasserts that selection across same-server route hydration or session reconciliation. A server-epoch change aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the replacement runtime guest. +- `PreviewAutomationHosts.tsx` resolves `browserDefaults.ts` once at the start of each automation-open request, before taking the session snapshot that pins a reused runtime guest. That single settings snapshot supplies the new-tab viewport and the automatic floating-preview preference, so creation and presentation cannot observe different settings during one request, while cold settings hydration cannot mix a pre-await session snapshot with a post-await server epoch. Explicit `open` or its deprecated `show` alias remains authoritative; when both are omitted, `autoShowFloatingPreview` decides presentation. The resulting `shouldPresentPreview` value is passed unchanged to `previewAutomationOpenReadiness.ts`, so a reused rendered tab left in the background does not wait for visibility while an explicitly shown tab does. A newly created tab applies its server snapshot and assigned tab id, uses the configured viewport or the branch's deterministic 1280×800 fallback when the snapshot remains `fill`, initiates any requested selection, and acknowledges server-side creation without depending on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; later wait, snapshot, or interaction operations own attachment and page readiness. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation and reasserts that selection across same-server route hydration or session reconciliation. A server-epoch change aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the replacement runtime guest. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - Retained browser guests subscribe only to render state for their own epoch-scoped runtime tab, plus their thread's active panel and mini-player selection. Presentation or background-capture updates for another tab do not rerender every mounted `HostedBrowserWebview`; selection changes override a stale surface-visible flag so background staging remains nearly transparent and exposes its readiness marker, and staging derives its viewport-fitted rectangle from the target tab's own stable rectangle. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. Multiple tokens received while an exchange is pending are serialized, and the submitting state remains active until every queued exchange settles. diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 9e6991f147a..7e2abdabf51 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -296,6 +296,8 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) }; let tabId = request.tabId ?? null; try { + const openBrowserDefaults = + request.operation === "open" ? await resolveBrowserDefaults() : undefined; let state = readThreadPreviewState(threadRef); const needsSessionSync = needsPreviewAutomationSessionSync(state, request.tabId); if (needsSessionSync) { @@ -347,10 +349,12 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) return await currentStatus(threadRef, tabId); case "open": { const input = request.input as PreviewAutomationOpenInput; - const browserDefaults = await resolveBrowserDefaults(); + if (!openBrowserDefaults) { + throw new Error("Browser defaults were not resolved for preview open"); + } const shouldPresentPreview = shouldOpenPreviewMiniPlayer( input, - browserDefaults.autoShowFloatingPreview, + openBrowserDefaults.autoShowFloatingPreview, ); const resolvedInputUrl = input.url ? resolveBrowserNavigationTarget(environmentId, { @@ -376,7 +380,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) ...(resolvedInputUrl ? { url: resolvedInputUrl } : {}), // An agent that didn't state a size gets the user's // configured default, same as a hand-opened tab. - viewport: browserDefaultOpenViewport(browserDefaults), + viewport: browserDefaultOpenViewport(openBrowserDefaults), }, }); if (result._tag === "Failure") { From fb2d11f0138c05ac48d47b6550ed3290935b16ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Tue, 18 Aug 2026 00:23:53 +0100 Subject: [PATCH 39/43] fix(web): bound preview mutations and viewport waits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - reject expired open mutations before starting side effects - clamp viewport polling and revalidate runtime identity - hide inactive capture guests from host assistive technology - document focused tests and integrated verification limits 🤖 Co-authored by GPT-5 in Codex via T3 Code --- BRANCH_DETAILS.md | 12 ++-- apps/web/src/browser/HostedBrowserWebview.tsx | 3 +- .../browser/hostedBrowserWebviewStyle.test.ts | 8 +++ .../src/browser/hostedBrowserWebviewStyle.ts | 4 ++ .../preview/PreviewAutomationHosts.tsx | 42 ++++++------ .../preview/previewViewportReadiness.test.ts | 68 ++++++++++++++++++- .../preview/previewViewportReadiness.ts | 31 +++++++++ 7 files changed, 137 insertions(+), 31 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 0d38309ba58..3bc631b3adc 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -27,9 +27,10 @@ Expected behavior: - A background snapshot that times out before desktop capture begins releases its presentation lease even when Chromium has paused compositor-frame callbacks. Once desktop capture starts, a timed-out snapshot retains its presentation lease until that capture settles, so response timeouts cannot tear down compositor staging beneath an in-flight capture. The desktop snapshot receives the operation's remaining timeout and bounds its control session accordingly. - The shared preview contract treats snapshot screenshots as nullable. MCP snapshot responses omit image content when capture is unavailable while preserving structured semantic content and explicitly reporting `screenshot: null`; tool descriptions promise a PNG only when capture is available. In `packages/contracts/src/ipc.ts`, the branch's desktop snapshot schema and upstream's create-tab defaults coexist: snapshot calls default an omitted `background` flag to `false`, while tab creation carries viewport, zoom, and color-scheme defaults through the preload and desktop manager. - The renderer automation consumer reserves response grace before the broker deadline and converts a stalled host operation into a typed `PreviewAutomationTimeoutError` instead of leaving the broker to surface a generic execution failure. Short caller-supplied timeouts retain their full execution budget, and the transition into grace reservation remains monotonic as requested timeouts increase. Requests that ask to open the inline preview use the request's remaining bounded visibility budget rather than a fixed two-second ceiling. Best-effort presentation settling uses a non-throwing remaining-budget read and clamps its 500-millisecond ceiling to that budget. Overlay status calls are themselves bounded by the remaining deadline and revalidate runtime guest identity after awaiting the desktop bridge, while overlay, navigation, and visibility polling clamp each sleep to the remaining deadline. Stable-presentation dwell also contracts to fit short deadlines instead of requiring an impossible fixed 100 milliseconds. Reused empty or failed tabs acknowledge without waiting for a browser surface those states intentionally hide. Visibility timeouts separately report the inline preview's selected tab, the right panel's active surface and open state, the active presentation kind, whether the requested browser surface was registered, and whether it had a presentation rectangle. -- `PreviewAutomationHosts.tsx` resolves `browserDefaults.ts` once at the start of each automation-open request, before taking the session snapshot that pins a reused runtime guest. That single settings snapshot supplies the new-tab viewport and the automatic floating-preview preference, so creation and presentation cannot observe different settings during one request, while cold settings hydration cannot mix a pre-await session snapshot with a post-await server epoch. Explicit `open` or its deprecated `show` alias remains authoritative; when both are omitted, `autoShowFloatingPreview` decides presentation. The resulting `shouldPresentPreview` value is passed unchanged to `previewAutomationOpenReadiness.ts`, so a reused rendered tab left in the background does not wait for visibility while an explicitly shown tab does. A newly created tab applies its server snapshot and assigned tab id, uses the configured viewport or the branch's deterministic 1280×800 fallback when the snapshot remains `fill`, initiates any requested selection, and acknowledges server-side creation without depending on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; later wait, snapshot, or interaction operations own attachment and page readiness. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation and reasserts that selection across same-server route hydration or session reconciliation. A server-epoch change aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the replacement runtime guest. +- `PreviewAutomationHosts.tsx` resolves `browserDefaults.ts` once at the start of each automation-open request, before taking the session snapshot that pins a reused runtime guest. That single settings snapshot supplies the new-tab viewport and the automatic floating-preview preference, so creation and presentation cannot observe different settings during one request, while cold settings hydration cannot mix a pre-await session snapshot with a post-await server epoch. Explicit `open` or its deprecated `show` alias remains authoritative; when both are omitted, `autoShowFloatingPreview` decides presentation. The resulting `shouldPresentPreview` value is passed unchanged to `previewAutomationOpenReadiness.ts`, so a reused rendered tab left in the background does not wait for visibility while an explicitly shown tab does. After settings and session synchronization, the host rechecks its remaining deadline immediately before tab creation and every later irreversible open-side mutation, so an already expired request cannot create, resize, reveal, or navigate a preview. A newly created tab applies its server snapshot and assigned tab id, uses the configured viewport or the branch's deterministic 1280×800 fallback when the snapshot remains `fill`, initiates any requested selection, and acknowledges server-side creation without depending on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; later wait, snapshot, or interaction operations own attachment and page readiness. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation and reasserts that selection across same-server route hydration or session reconciliation. A server-epoch change aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the replacement runtime guest. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. -- Retained browser guests subscribe only to render state for their own epoch-scoped runtime tab, plus their thread's active panel and mini-player selection. Presentation or background-capture updates for another tab do not rerender every mounted `HostedBrowserWebview`; selection changes override a stale surface-visible flag so background staging remains nearly transparent and exposes its readiness marker, and staging derives its viewport-fitted rectangle from the target tab's own stable rectangle. +- Retained browser guests subscribe only to render state for their own epoch-scoped runtime tab, plus their thread's active panel and mini-player selection. Presentation or background-capture updates for another tab do not rerender every mounted `HostedBrowserWebview`; selection changes override a stale surface-visible flag so background staging remains nearly transparent and exposes its readiness marker, and staging derives its viewport-fitted rectangle from the target tab's own stable rectangle. Only the active guest is exposed through the host accessibility tree; a nearly transparent background-capture guest stays `aria-hidden` without affecting CDP accessibility-tree collection against the guest target. +- Rendered viewport readiness revalidates the epoch-scoped runtime guest after each awaited guest measurement and clamps every polling sleep to the remaining resize deadline, preserving typed target-replacement and viewport-timeout failures for short budgets. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. Multiple tokens received while an exchange is pending are serialized, and the submitting state remains active until every queued exchange settles. - `dev:desktop` derives `T3CODE_DESKTOP_USER_DATA_DIR=/userdata/electron` whenever the runner has an explicit base directory. Desktop configuration resolves that override to an absolute path, and app identity uses it before legacy migration or the normal Electron user-data default. This keeps an isolated worktree dev desktop from reusing an installed or earlier development profile whose incompatible IndexedDB schema can prevent the renderer from starting. Packaged/default startup remains unchanged when no override is supplied. @@ -66,18 +67,19 @@ Primary files: - `packages/contracts/src/ipc.ts` - `scripts/dev-runner.ts` -Focused regression coverage lives in `scripts/dev-runner.test.ts`, `apps/desktop/src/app/DesktopAppIdentity.test.ts`, `apps/desktop/src/app/DesktopEnvironment.test.ts`, `apps/desktop/src/preview/Manager.test.ts`, `apps/server/src/mcp/McpHttpServer.test.ts`, `apps/web/src/browser/browserSurfaceStore.test.ts`, `apps/web/src/browser/hostedBrowserWebviewStyle.test.ts`, `apps/web/src/browser/browserViewportActions.test.ts`, `apps/web/src/browser/browserViewportLayout.test.ts`, `apps/web/src/browser/previewRuntimeTabId.test.ts`, `apps/web/src/components/auth/PairingRouteSurface.logic.test.ts`, `apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationPresentation.test.ts`, `apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts`, `apps/web/src/components/preview/previewNavigationReadiness.test.ts`, `apps/web/src/components/preview/previewViewportRollback.test.ts`, `packages/contracts/src/ipc.test.ts`, and `packages/contracts/src/preview.test.ts`. +Focused regression coverage lives in `scripts/dev-runner.test.ts`, `apps/desktop/src/app/DesktopAppIdentity.test.ts`, `apps/desktop/src/app/DesktopEnvironment.test.ts`, `apps/desktop/src/preview/Manager.test.ts`, `apps/server/src/mcp/McpHttpServer.test.ts`, `apps/web/src/browser/browserSurfaceStore.test.ts`, `apps/web/src/browser/hostedBrowserWebviewStyle.test.ts`, `apps/web/src/browser/browserViewportActions.test.ts`, `apps/web/src/browser/browserViewportLayout.test.ts`, `apps/web/src/browser/previewRuntimeTabId.test.ts`, `apps/web/src/components/auth/PairingRouteSurface.logic.test.ts`, `apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationPresentation.test.ts`, `apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts`, `apps/web/src/components/preview/previewNavigationReadiness.test.ts`, `apps/web/src/components/preview/previewViewportReadiness.test.ts`, `apps/web/src/components/preview/previewViewportRollback.test.ts`, `packages/contracts/src/ipc.test.ts`, and `packages/contracts/src/preview.test.ts`. ```sh -vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.test.ts apps/desktop/src/app/DesktopEnvironment.test.ts apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/web/src/browser/browserSurfaceStore.test.ts apps/web/src/browser/hostedBrowserWebviewStyle.test.ts apps/web/src/browser/browserViewportActions.test.ts apps/web/src/browser/browserViewportLayout.test.ts apps/web/src/browser/previewRuntimeTabId.test.ts apps/web/src/components/auth/PairingRouteSurface.logic.test.ts apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts apps/web/src/components/preview/previewAutomationPresentation.test.ts apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts apps/web/src/components/preview/previewNavigationReadiness.test.ts apps/web/src/components/preview/previewViewportRollback.test.ts packages/contracts/src/ipc.test.ts packages/contracts/src/preview.test.ts +vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.test.ts apps/desktop/src/app/DesktopEnvironment.test.ts apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/web/src/browser/browserSurfaceStore.test.ts apps/web/src/browser/hostedBrowserWebviewStyle.test.ts apps/web/src/browser/browserViewportActions.test.ts apps/web/src/browser/browserViewportLayout.test.ts apps/web/src/browser/previewRuntimeTabId.test.ts apps/web/src/components/auth/PairingRouteSurface.logic.test.ts apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts apps/web/src/components/preview/previewAutomationPresentation.test.ts apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts apps/web/src/components/preview/previewNavigationReadiness.test.ts apps/web/src/components/preview/previewViewportReadiness.test.ts apps/web/src/components/preview/previewViewportRollback.test.ts packages/contracts/src/ipc.test.ts packages/contracts/src/preview.test.ts ``` Current verification: -- The branch-focused suite passed 254 tests across 17 files on Windows. The two unchanged desktop path-fixture files in the command above currently produce eight POSIX-versus-Windows path assertion failures; the affected runtime behavior passes in the remaining focused suite. +- The branch-focused suite passed 261 tests across 18 files on Windows, including the new short-deadline viewport polling, post-read runtime-replacement, and background-guest accessibility regressions. The two unchanged desktop path-fixture files in the command above currently produce eight POSIX-versus-Windows path assertion failures; the affected runtime behavior passes in the remaining focused suite. - The incoming browser-default, provider-access, pull-request budget, tooltip, update-copy, and settings coverage passed 552 tests across 26 files. The focused open-policy and open-session subset passes 18 tests, including a shared explicit-over-default presentation decision and a reused rendered tab that remains in the background without a visibility wait. - Desktop, server, contracts, and mobile typechecks pass. Web typecheck currently reports 16 existing errors in `BranchToolbarBranchSelector.tsx`, `ModelPickerContent.tsx`, `PreviewAutomationHosts.tsx` at the pre-existing registry access, `FontFamilyPicker.tsx`, `use-atom-command.ts`, and `use-atom-query-runner.ts`; none are in the browser-default integration changed here. - An isolated web client on ports `5744`/`13784` paired and loaded successfully. Settings → Integrations exposed the browser defaults and agent-access setting; agent browser access could be disabled and restored. The non-Electron client correctly disabled desktop-only viewport, zoom, and appearance controls. +- The review-follow-up isolated stack on ports `5744`/`13784` served the React app shell with HTTP 200. No current integrated UI result is available for the follow-up changes because the in-app browser inventory was empty, Playwright was not installed, and the Windows control pipe was unavailable; this is a verification-harness blocker rather than a product failure or pass. - Android paired with an isolated server, loaded the seeded project, created a thread, and received a response. The changed Android thread-settings header remains visually unverified because Metro first exhausted file handles and a clean restart then exposed an installed-dev-client `EventEmitter` runtime mismatch. ## Development Ports diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index 7e85d4227bd..a4bc0e990e8 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -24,6 +24,7 @@ import { BrowserDeviceToolbar } from "./BrowserDeviceToolbar"; import { BrowserViewportResizeHandles } from "./BrowserViewportResizeHandles"; import { acquireDesktopTab, type AcquiredDesktopTab } from "./desktopTabLifetime"; import { + resolveHostedBrowserWebviewAriaHidden, resolveHostedBrowserWebviewPresentation, resolveHostedBrowserWebviewWrapperStyle, } from "./hostedBrowserWebviewStyle"; @@ -308,7 +309,7 @@ export function HostedBrowserWebview(props: { ? Math.max(1, Math.round(layout.viewportHeight / normalizedZoomFactor)) : effectiveViewport.height } - aria-hidden={active || backgroundCapture ? undefined : true} + aria-hidden={resolveHostedBrowserWebviewAriaHidden(active)} className={cn( "absolute flex overflow-hidden bg-background", active && !layout.fillsPanel && "ring-1 ring-border/70 shadow-sm", diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts index 87bf1aa269f..ee329fa2e47 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts @@ -4,10 +4,18 @@ import { BACKGROUND_CAPTURE_BROWSER_WEBVIEW_OPACITY, BACKGROUND_CAPTURE_BROWSER_WEBVIEW_Z_INDEX, HIDDEN_BROWSER_WEBVIEW_OFFSET, + resolveHostedBrowserWebviewAriaHidden, resolveHostedBrowserWebviewPresentation, resolveHostedBrowserWebviewWrapperStyle, } from "./hostedBrowserWebviewStyle"; +describe("resolveHostedBrowserWebviewAriaHidden", () => { + it("exposes only the active guest to host assistive technology", () => { + expect(resolveHostedBrowserWebviewAriaHidden(true)).toBeUndefined(); + expect(resolveHostedBrowserWebviewAriaHidden(false)).toBe(true); + }); +}); + describe("resolveHostedBrowserWebviewPresentation", () => { it("stages a background capture when visibility is stale after selection changes", () => { expect( diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.ts index b322b39675d..d37b912bc00 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.ts @@ -21,6 +21,10 @@ export const HIDDEN_BROWSER_WEBVIEW_OFFSET = -100_000; export const BACKGROUND_CAPTURE_BROWSER_WEBVIEW_Z_INDEX = 31; export const BACKGROUND_CAPTURE_BROWSER_WEBVIEW_OPACITY = 0.001; +export function resolveHostedBrowserWebviewAriaHidden(active: boolean): true | undefined { + return active ? undefined : true; +} + export function resolveHostedBrowserWebviewPresentation(input: { readonly backgroundCaptureRequested: boolean; readonly hasRect: boolean; diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 7e2abdabf51..d844b5314e5 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -83,7 +83,7 @@ import { resolvePreviewAutomationOpenTab, resolvePreviewAutomationTarget, } from "./previewAutomationTarget"; -import { isPreviewViewportReady } from "./previewViewportReadiness"; +import { waitForPreviewViewportReadiness } from "./previewViewportReadiness"; import { shouldRollbackPreviewViewport } from "./previewViewportRollback"; interface ExecutablePreviewWebview extends Element { @@ -144,31 +144,23 @@ const waitForRenderedViewport = async ( readonly threadId: PreviewAutomationRequest["threadId"]; }, ): Promise => { - const deadline = Date.now() + timeoutMs; - while (Date.now() <= deadline) { - assertPreviewRuntimeCurrent(threadRef, tabId, runtimeTabId, context); - try { + const renderedViewport = await waitForPreviewViewportReadiness({ + setting, + timeoutMs, + assertCurrent: () => assertPreviewRuntimeCurrent(threadRef, tabId, runtimeTabId, context), + readViewport: async () => { const webview = findPreviewWebview(runtimeTabId); + if (!webview) return null; const appliedSettingKey = webview?.getAttribute("data-preview-viewport-key") ?? null; const declaredViewport = readDeclaredViewport(webview); - const renderedViewport = webview ? await readWebviewViewport(webview) : null; - if ( - renderedViewport && - isPreviewViewportReady({ - setting, - appliedSettingKey, - declaredViewport, - renderedViewport, - }) - ) { - return renderedViewport; - } - } catch { - // Registration and navigation can transiently replace the guest while - // React applies the server snapshot. Retry until the operation deadline. - } - await new Promise((resolve) => window.setTimeout(resolve, 50)); - } + return { + appliedSettingKey, + declaredViewport, + renderedViewport: await readWebviewViewport(webview), + }; + }, + }); + if (renderedViewport) return renderedViewport; throw new PreviewAutomationViewportTimeoutError({ ...context, tabId, @@ -373,6 +365,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) const reusedExistingTab = activeTabId !== null; tabId = activeTabId; if (!activeTabId) { + remainingOperationBudget(); const result = await open({ environmentId, input: { @@ -412,6 +405,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) activeRuntimeTabId, request, ); + remainingOperationBudget(); return await resize({ environmentId, input: { @@ -430,6 +424,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } } if (shouldPresentPreview) { + remainingOperationBudget(); revealPreviewAutomationTab(threadRef, activeTabId); } const waitPolicy = activeSnapshot @@ -465,6 +460,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } if (reusedExistingTab && resolvedInputUrl && previewBridge) { assertPreviewRuntimeCurrent(threadRef, activeTabId, activeRuntimeTabId, request); + remainingOperationBudget(); await previewBridge.navigate(activeRuntimeTabId, resolvedInputUrl); await waitForNavigationReadiness( threadRef, diff --git a/apps/web/src/components/preview/previewViewportReadiness.test.ts b/apps/web/src/components/preview/previewViewportReadiness.test.ts index c47ffc3ff08..f1a9ff2e1af 100644 --- a/apps/web/src/components/preview/previewViewportReadiness.test.ts +++ b/apps/web/src/components/preview/previewViewportReadiness.test.ts @@ -1,8 +1,11 @@ -import { describe, expect, it } from "vite-plus/test"; +import { describe, expect, it, vi } from "vite-plus/test"; import { browserViewportSettingKey } from "~/browser/browserViewportLayout"; -import { isPreviewViewportReady } from "./previewViewportReadiness"; +import { + isPreviewViewportReady, + waitForPreviewViewportReadiness, +} from "./previewViewportReadiness"; describe("isPreviewViewportReady", () => { const landscape = { @@ -70,3 +73,64 @@ describe("isPreviewViewportReady", () => { ).toBe(false); }); }); + +describe("waitForPreviewViewportReadiness", () => { + const setting = { + _tag: "preset", + width: 844, + height: 390, + presetId: "iphone-12-pro", + } as const; + const readyViewport = { + appliedSettingKey: browserViewportSettingKey(setting), + declaredViewport: { width: 844, height: 390 }, + renderedViewport: { width: 844, height: 390 }, + } as const; + + it("clamps polling to the remaining short deadline", async () => { + vi.useFakeTimers(); + vi.stubGlobal("window", { setTimeout }); + try { + const result = waitForPreviewViewportReadiness({ + setting, + timeoutMs: 40, + assertCurrent: vi.fn(), + readViewport: vi.fn().mockResolvedValue(null), + }); + + await vi.advanceTimersByTimeAsync(40); + await expect(result).resolves.toBeNull(); + } finally { + vi.unstubAllGlobals(); + vi.useRealTimers(); + } + }); + + it("revalidates runtime identity after an awaited viewport read", async () => { + let resolveViewport!: (value: typeof readyViewport) => void; + const readViewport = vi.fn( + () => + new Promise((resolve) => { + resolveViewport = resolve; + }), + ); + const replaced = new Error("runtime replaced"); + const assertCurrent = vi + .fn() + .mockImplementationOnce(() => undefined) + .mockImplementation(() => { + throw replaced; + }); + const result = waitForPreviewViewportReadiness({ + setting, + timeoutMs: 1_000, + assertCurrent, + readViewport, + }); + + resolveViewport(readyViewport); + + await expect(result).rejects.toBe(replaced); + expect(assertCurrent).toHaveBeenCalledTimes(2); + }); +}); diff --git a/apps/web/src/components/preview/previewViewportReadiness.ts b/apps/web/src/components/preview/previewViewportReadiness.ts index 5ff963fbceb..af069c4cfc9 100644 --- a/apps/web/src/components/preview/previewViewportReadiness.ts +++ b/apps/web/src/components/preview/previewViewportReadiness.ts @@ -35,3 +35,34 @@ export function isPreviewViewportReady(input: { Math.abs(renderedViewport.height - expectedViewport.height) <= tolerance ); } + +export async function waitForPreviewViewportReadiness(input: { + readonly setting: PreviewViewportSetting; + readonly timeoutMs: number; + readonly assertCurrent: () => void; + readonly readViewport: () => Promise<{ + readonly appliedSettingKey: string | null; + readonly declaredViewport: PreviewRenderedViewportSize | null; + readonly renderedViewport: PreviewRenderedViewportSize | null; + } | null>; +}): Promise { + const deadline = Date.now() + input.timeoutMs; + while (Date.now() <= deadline) { + input.assertCurrent(); + let viewportState: Awaited> = null; + try { + viewportState = await input.readViewport(); + } catch { + // Registration and navigation can transiently replace the guest while + // React applies the server snapshot. Retry until the operation deadline. + } + input.assertCurrent(); + if (viewportState && isPreviewViewportReady({ setting: input.setting, ...viewportState })) { + return viewportState.renderedViewport; + } + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) break; + await new Promise((resolve) => window.setTimeout(resolve, Math.min(50, remainingMs))); + } + return null; +} From d09aa77c80ab7dcd48697f03ee54e5e1126585c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Tue, 18 Aug 2026 00:50:43 +0100 Subject: [PATCH 40/43] fix(preview): bound post-readiness mutations --- BRANCH_DETAILS.md | 11 +-- apps/desktop/src/ipc/methods/preview.ts | 23 +++++-- apps/desktop/src/preload.ts | 12 ++-- apps/desktop/src/preview/Manager.test.ts | 67 +++++++++++++++++++ apps/desktop/src/preview/Manager.ts | 60 +++++++++++++++-- apps/server/src/mcp/McpHttpServer.test.ts | 7 +- .../src/mcp/toolkits/preview/handlers.ts | 14 ++-- apps/server/src/mcp/toolkits/preview/tools.ts | 3 +- apps/web/src/browser/browserRecording.test.ts | 20 ++++++ apps/web/src/browser/browserRecording.ts | 31 +++++++-- .../preview/PreviewAutomationHosts.tsx | 21 +++++- packages/contracts/src/ipc.ts | 18 ++++- packages/contracts/src/preview.test.ts | 19 ++++++ packages/contracts/src/previewAutomation.ts | 9 +++ 14 files changed, 274 insertions(+), 41 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 3bc631b3adc..67b0fb0b25d 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -20,7 +20,7 @@ This branch does not add a second recording/PiP capture lifecycle or another hid Expected behavior: - Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout without making the execution budget shrink when the caller increases a short timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session while still holding an acquired control permit when a CDP command may be pending. Session removal and debugger teardown are atomic with respect to new session acquisition and bound to the exact acquired session, so late interruption or snapshot cleanup cannot detach a healthy replacement. Operations already queued on the retired semaphore detect that stale session and retry against its replacement. A request that times out while queued behind another action does not detach that action's shared debugger session. -- Click, type, and wait operations clamp their caller-supplied timeout to the remaining renderer host budget before entering the desktop control-session boundary. Operations without a caller timeout use the remaining bounded request budget rather than restarting the desktop default after renderer readiness work. +- Click, type, press, scroll, and wait operations clamp their caller-supplied timeout to the remaining renderer host budget before entering the desktop control-session boundary. Color-scheme changes and recording startup likewise receive the remaining deadline after overlay readiness; a timed-out color-scheme command does not persist a late preference, and timed-out recording startup tears down its frame-capture session and renderer recording state. Operations without a caller timeout use the remaining bounded request budget rather than restarting the desktop default after renderer readiness work. - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Primary and fallback screenshot waits are clamped to the remaining control-session deadline, with budget reserved for fallback and result settlement, so a tight caller deadline can still return semantic data instead of being preempted by the outer session timeout. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, an actually timed-out CDP capture resets the session before releasing its control permit, queued work reattaches before issuing its first command, and a capture skipped before CDP runs leaves the healthy session attached. The semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. `apps/web/src/browser/desktopTabLifetime.ts` passes the upstream browser appearance default through `DesktopPreviewCreateTabInputSchema` in `packages/contracts/src/ipc.ts`; `apps/desktop/src/preview/Manager.ts` normalizes that value. A non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path, while tabs following the system scheme stay detached until the next automation operation. - Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the server epoch replaces the runtime guest while staging is pending, the snapshot fails immediately with `PreviewAutomationTargetUnavailableError` instead of waiting on the stale staging marker. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. @@ -50,6 +50,7 @@ Primary files: - `apps/server/src/mcp/McpHttpServer.ts` - `apps/server/src/mcp/toolkits/preview/tools.ts` - `apps/web/src/browser/HostedBrowserWebview.tsx` +- `apps/web/src/browser/browserRecording.ts` - `apps/web/src/browser/browserDefaults.ts` - `apps/web/src/browser/browserSurfaceStore.ts` - `apps/web/src/browser/desktopTabLifetime.ts` @@ -67,19 +68,19 @@ Primary files: - `packages/contracts/src/ipc.ts` - `scripts/dev-runner.ts` -Focused regression coverage lives in `scripts/dev-runner.test.ts`, `apps/desktop/src/app/DesktopAppIdentity.test.ts`, `apps/desktop/src/app/DesktopEnvironment.test.ts`, `apps/desktop/src/preview/Manager.test.ts`, `apps/server/src/mcp/McpHttpServer.test.ts`, `apps/web/src/browser/browserSurfaceStore.test.ts`, `apps/web/src/browser/hostedBrowserWebviewStyle.test.ts`, `apps/web/src/browser/browserViewportActions.test.ts`, `apps/web/src/browser/browserViewportLayout.test.ts`, `apps/web/src/browser/previewRuntimeTabId.test.ts`, `apps/web/src/components/auth/PairingRouteSurface.logic.test.ts`, `apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationPresentation.test.ts`, `apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts`, `apps/web/src/components/preview/previewNavigationReadiness.test.ts`, `apps/web/src/components/preview/previewViewportReadiness.test.ts`, `apps/web/src/components/preview/previewViewportRollback.test.ts`, `packages/contracts/src/ipc.test.ts`, and `packages/contracts/src/preview.test.ts`. +Focused regression coverage lives in `scripts/dev-runner.test.ts`, `apps/desktop/src/app/DesktopAppIdentity.test.ts`, `apps/desktop/src/app/DesktopEnvironment.test.ts`, `apps/desktop/src/preview/Manager.test.ts`, `apps/server/src/mcp/McpHttpServer.test.ts`, `apps/web/src/browser/browserRecording.test.ts`, `apps/web/src/browser/browserSurfaceStore.test.ts`, `apps/web/src/browser/hostedBrowserWebviewStyle.test.ts`, `apps/web/src/browser/browserViewportActions.test.ts`, `apps/web/src/browser/browserViewportLayout.test.ts`, `apps/web/src/browser/previewRuntimeTabId.test.ts`, `apps/web/src/components/auth/PairingRouteSurface.logic.test.ts`, `apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts`, `apps/web/src/components/preview/previewAutomationPresentation.test.ts`, `apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts`, `apps/web/src/components/preview/previewNavigationReadiness.test.ts`, `apps/web/src/components/preview/previewViewportReadiness.test.ts`, `apps/web/src/components/preview/previewViewportRollback.test.ts`, `packages/contracts/src/ipc.test.ts`, and `packages/contracts/src/preview.test.ts`. ```sh -vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.test.ts apps/desktop/src/app/DesktopEnvironment.test.ts apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/web/src/browser/browserSurfaceStore.test.ts apps/web/src/browser/hostedBrowserWebviewStyle.test.ts apps/web/src/browser/browserViewportActions.test.ts apps/web/src/browser/browserViewportLayout.test.ts apps/web/src/browser/previewRuntimeTabId.test.ts apps/web/src/components/auth/PairingRouteSurface.logic.test.ts apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts apps/web/src/components/preview/previewAutomationPresentation.test.ts apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts apps/web/src/components/preview/previewNavigationReadiness.test.ts apps/web/src/components/preview/previewViewportReadiness.test.ts apps/web/src/components/preview/previewViewportRollback.test.ts packages/contracts/src/ipc.test.ts packages/contracts/src/preview.test.ts +vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.test.ts apps/desktop/src/app/DesktopEnvironment.test.ts apps/desktop/src/preview/Manager.test.ts apps/server/src/mcp/McpHttpServer.test.ts apps/web/src/browser/browserRecording.test.ts apps/web/src/browser/browserSurfaceStore.test.ts apps/web/src/browser/hostedBrowserWebviewStyle.test.ts apps/web/src/browser/browserViewportActions.test.ts apps/web/src/browser/browserViewportLayout.test.ts apps/web/src/browser/previewRuntimeTabId.test.ts apps/web/src/components/auth/PairingRouteSurface.logic.test.ts apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts apps/web/src/components/preview/previewAutomationOverlayReadiness.test.ts apps/web/src/components/preview/previewAutomationPresentation.test.ts apps/web/src/components/preview/previewAutomationRequestConsumer.test.ts apps/web/src/components/preview/previewNavigationReadiness.test.ts apps/web/src/components/preview/previewViewportReadiness.test.ts apps/web/src/components/preview/previewViewportRollback.test.ts packages/contracts/src/ipc.test.ts packages/contracts/src/preview.test.ts ``` Current verification: -- The branch-focused suite passed 261 tests across 18 files on Windows, including the new short-deadline viewport polling, post-read runtime-replacement, and background-guest accessibility regressions. The two unchanged desktop path-fixture files in the command above currently produce eight POSIX-versus-Windows path assertion failures; the affected runtime behavior passes in the remaining focused suite. +- The branch-focused suite passed 278 tests across 19 files on Windows, including the new bounded post-overlay mutations, recording-start cleanup, short-deadline viewport polling, post-read runtime-replacement, and background-guest accessibility regressions. The two unchanged desktop path-fixture files in the command above currently produce eight POSIX-versus-Windows path assertion failures; the affected runtime behavior passes in the remaining focused suite. - The incoming browser-default, provider-access, pull-request budget, tooltip, update-copy, and settings coverage passed 552 tests across 26 files. The focused open-policy and open-session subset passes 18 tests, including a shared explicit-over-default presentation decision and a reused rendered tab that remains in the background without a visibility wait. - Desktop, server, contracts, and mobile typechecks pass. Web typecheck currently reports 16 existing errors in `BranchToolbarBranchSelector.tsx`, `ModelPickerContent.tsx`, `PreviewAutomationHosts.tsx` at the pre-existing registry access, `FontFamilyPicker.tsx`, `use-atom-command.ts`, and `use-atom-query-runner.ts`; none are in the browser-default integration changed here. - An isolated web client on ports `5744`/`13784` paired and loaded successfully. Settings → Integrations exposed the browser defaults and agent-access setting; agent browser access could be disabled and restored. The non-Electron client correctly disabled desktop-only viewport, zoom, and appearance controls. -- The review-follow-up isolated stack on ports `5744`/`13784` served the React app shell with HTTP 200. No current integrated UI result is available for the follow-up changes because the in-app browser inventory was empty, Playwright was not installed, and the Windows control pipe was unavailable; this is a verification-harness blocker rather than a product failure or pass. +- The final review-follow-up isolated stack on ports `5744`/`13784` paired through Playwright, loaded the React app shell, and reported no browser-console errors. The affected mutation paths remain Electron-only and therefore cannot execute in the non-Electron web client; their deadline propagation and cleanup are covered by the focused renderer, contracts, server, and desktop tests rather than an end-to-end web-browser assertion. - Android paired with an isolated server, loaded the seeded project, created a thread, and received a response. The changed Android thread-settings header remains visually unverified because Metro first exhausted file handles and a clean restart then exposed an installed-dev-client `EventEmitter` runtime mismatch. ## Development Ports diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 91d12a61f61..0fb40ea128f 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -11,6 +11,7 @@ import { DesktopPreviewConfigInputSchema, DesktopPreviewNavigateInputSchema, DesktopPreviewRecordingArtifactSchema, + DesktopPreviewRecordingStartInputSchema, DesktopPreviewRecordingSaveInputSchema, DesktopPreviewRegisterWebviewInputSchema, DesktopPreviewScreenshotArtifactSchema, @@ -149,9 +150,13 @@ export const setColorScheme = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PREVIEW_SET_COLOR_SCHEME_CHANNEL, payload: DesktopPreviewSetColorSchemeInputSchema, result: Schema.Void, - handler: Effect.fn("desktop.ipc.preview.setColorScheme")(function* ({ tabId, colorScheme }) { + handler: Effect.fn("desktop.ipc.preview.setColorScheme")(function* ({ + tabId, + colorScheme, + timeoutMs, + }) { const manager = yield* PreviewManager.PreviewManager; - yield* manager.setColorScheme(tabId, colorScheme); + yield* manager.setColorScheme(tabId, colorScheme, timeoutMs); }), }); export const openDevTools = tabMethod( @@ -164,11 +169,15 @@ export const cancelPickElement = tabMethod( "desktop.ipc.preview.cancelPickElement", (manager, tabId) => manager.cancelPickElement(tabId), ); -export const startRecording = tabMethod( - IpcChannels.PREVIEW_RECORDING_START_CHANNEL, - "desktop.ipc.preview.startRecording", - (manager, tabId) => manager.startRecording(tabId), -); +export const startRecording = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.PREVIEW_RECORDING_START_CHANNEL, + payload: DesktopPreviewRecordingStartInputSchema, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.preview.startRecording")(function* ({ tabId, timeoutMs }) { + const manager = yield* PreviewManager.PreviewManager; + yield* manager.startRecording(tabId, timeoutMs); + }), +}); export const stopRecording = tabMethod( IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL, "desktop.ipc.preview.stopRecording", diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index a2fa5fe0758..b11ed469f89 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -181,8 +181,12 @@ contextBridge.exposeInMainWorld("desktopBridge", { zoomOut: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_ZOOM_OUT_CHANNEL, { tabId }), resetZoom: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_RESET_ZOOM_CHANNEL, { tabId }), hardReload: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_HARD_RELOAD_CHANNEL, { tabId }), - setColorScheme: (tabId, colorScheme) => - ipcRenderer.invoke(IpcChannels.PREVIEW_SET_COLOR_SCHEME_CHANNEL, { tabId, colorScheme }), + setColorScheme: (tabId, colorScheme, timeoutMs) => + ipcRenderer.invoke(IpcChannels.PREVIEW_SET_COLOR_SCHEME_CHANNEL, { + tabId, + colorScheme, + timeoutMs, + }), openDevTools: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL, { tabId }), clearCookies: () => ipcRenderer.invoke(IpcChannels.PREVIEW_CLEAR_COOKIES_CHANNEL), @@ -207,8 +211,8 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.invoke(IpcChannels.PREVIEW_PICTURE_IN_PICTURE_CLOSE_CHANNEL, { tabId }), }, recording: { - startScreencast: (tabId) => - ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_START_CHANNEL, { tabId }), + startScreencast: (tabId, timeoutMs) => + ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_START_CHANNEL, { tabId, timeoutMs }), stopScreencast: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL, { tabId }), save: (tabId, mimeType, data) => diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 1551c9aa9f1..e7e9dc4534b 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1721,6 +1721,73 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("does not persist a color-scheme mutation after its deadline", () => + withManager((manager) => + Effect.gen(function* () { + let attached = false; + const attach = vi.fn(() => { + attached = true; + }); + const detach = vi.fn(() => { + attached = false; + }); + const sendCommand = vi.fn(() => new Promise(() => undefined)); + fromId.mockReturnValue({ + id: 42, + isDestroyed: () => false, + isDevToolsOpened: () => false, + getType: () => "webview", + getURL: () => "https://example.com", + getTitle: () => "Example", + isLoading: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => attached, + attach, + detach, + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + } as never); + const states: PreviewManager.PreviewTabState[] = []; + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + states.push(state); + }), + ); + yield* manager.createTab("tab_scheme_timeout"); + yield* manager.registerWebview("tab_scheme_timeout", 42); + yield* Effect.yieldNow; + + const mutation = yield* manager + .setColorScheme("tab_scheme_timeout", "dark", 100) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + yield* TestClock.adjust(100); + const result = yield* Effect.exit(Fiber.join(mutation)); + + expect(Exit.isFailure(result)).toBe(true); + if (Exit.isFailure(result)) { + expect(Option.getOrThrow(Cause.findErrorOption(result.cause))).toMatchObject({ + _tag: "PreviewAutomationTimeoutError", + tabId: "tab_scheme_timeout", + timeoutMs: 100, + }); + } + expect(states.at(-1)?.colorScheme).toBe("system"); + expect(detach).toHaveBeenCalledOnce(); + }), + ), + ); + effectIt.effect("blocks late webview and capture starts during tab close", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 1d0011fd5d2..1b8b91606b5 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2331,11 +2331,34 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const setColorScheme = Effect.fn("PreviewManager.setColorScheme")(function* ( tabId: string, colorScheme: DesktopPreviewColorScheme, + timeoutMs?: number, ) { const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); if (!tab) { return yield* new PreviewTabNotFoundError({ tabId }); } + if (timeoutMs !== undefined) { + const wc = yield* requireWebContents(tabId); + yield* withControlSession( + tabId, + wc, + "set-color-scheme", + (send) => + send("Emulation.setEmulatedMedia", { + features: [ + { + name: "prefers-color-scheme", + value: colorScheme === "system" ? "" : colorScheme, + }, + ], + }), + timeoutMs, + ); + if (tab.colorScheme !== colorScheme) { + yield* update(tabId, { colorScheme }); + } + return; + } if (tab.colorScheme !== colorScheme) { // Record the choice even when the CDP call below can't run yet (no // webview, DevTools holding the debugger) — it is re-applied on the @@ -2880,8 +2903,19 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return yield* Effect.failCause(initializationExit.cause); }); - const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) { - yield* startFrameCapture(tabId, "recording"); + const startRecording = Effect.fn("PreviewManager.startRecording")(function* ( + tabId: string, + timeoutMs?: number, + ) { + const start = startFrameCapture(tabId, "recording"); + if (timeoutMs === undefined) { + yield* start; + return; + } + const result = yield* start.pipe(Effect.timeoutOption(automationExecutionBudget(timeoutMs))); + if (Option.isSome(result)) return; + yield* stopFrameCapture(tabId, "recording").pipe(Effect.ignore); + return yield* new PreviewAutomationTimeoutError({ tabId, timeoutMs }); }); const stopRecording = Effect.fn("PreviewManager.stopRecording")(function* (tabId: string) { @@ -3701,8 +3735,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function input: PreviewAutomationPressInput, ) { const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "press", (send, sendCleanup) => - performAutomationPress(tabId, wc, input, send, sendCleanup), + yield* withControlSession( + tabId, + wc, + "press", + (send, sendCleanup) => performAutomationPress(tabId, wc, input, send, sendCleanup), + input.timeoutMs, ); }); @@ -3757,8 +3795,12 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function input: PreviewAutomationScrollInput, ) { const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "scroll", (send) => - performAutomationScroll(tabId, input, send), + yield* withControlSession( + tabId, + wc, + "scroll", + (send) => performAutomationScroll(tabId, input, send), + input.timeoutMs, ); }); @@ -4259,6 +4301,7 @@ export class PreviewManager extends Context.Service< readonly setColorScheme: ( tabId: string, colorScheme: DesktopPreviewColorScheme, + timeoutMs?: number, ) => Effect.Effect; readonly openDevTools: (tabId: string) => Effect.Effect; readonly clearCookies: () => Effect.Effect; @@ -4278,7 +4321,10 @@ export class PreviewManager extends Context.Service< readonly copyArtifactToClipboard: (path: string) => Effect.Effect; readonly openPictureInPicture: (tabId: string) => Effect.Effect; readonly closePictureInPicture: (tabId: string) => Effect.Effect; - readonly startRecording: (tabId: string) => Effect.Effect; + readonly startRecording: ( + tabId: string, + timeoutMs?: number, + ) => Effect.Effect; readonly stopRecording: (tabId: string) => Effect.Effect; readonly saveRecording: ( tabId: string, diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index 5aaba3d8c87..d8668fe9a46 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -160,6 +160,7 @@ it.effect("registers annotated tools and preserves authenticated request context const routedRequests: Array<{ readonly operation: string; readonly tabId?: string | undefined; + readonly timeoutMs?: number | undefined; }> = []; const events = yield* broker.connect({ clientId: "mcp-test-client", @@ -282,8 +283,8 @@ it.effect("registers annotated tools and preserves authenticated request context const actionRequests = [ { name: "preview_click", arguments: { x: 10, y: 10 } }, { name: "preview_type", arguments: { text: "Hello" } }, - { name: "preview_press", arguments: { key: "Enter" } }, - { name: "preview_scroll", arguments: { deltaY: 100 } }, + { name: "preview_press", arguments: { key: "Enter", timeoutMs: 1_234 } }, + { name: "preview_scroll", arguments: { deltaY: 100, timeoutMs: 1_234 } }, { name: "preview_wait_for", arguments: { text: "Example" } }, ]; for (const request of actionRequests) { @@ -297,6 +298,8 @@ it.effect("registers annotated tools and preserves authenticated request context expect(result.structuredContent).toEqual({}); expect(result.content).toEqual([{ type: "text", text: "{}" }]); } + expect(routedRequests.find(({ operation }) => operation === "press")?.timeoutMs).toBe(1_234); + expect(routedRequests.find(({ operation }) => operation === "scroll")?.timeoutMs).toBe(1_234); }), ).pipe(Effect.provide(TestLayer)), ); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index 17da7801488..567a28e23c1 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -76,19 +76,25 @@ const handlers = { preview_resize: (input) => invokeTargeted("resize", input, input.timeoutMs), preview_set_appearance: (input) => - invokeTargeted("setColorScheme", input), + invokeTargeted("setColorScheme", input, input.timeoutMs), preview_snapshot: (input) => invokeTargeted("snapshot", input ?? {}), preview_click: (input) => invokeTargeted("click", input, input.timeoutMs).pipe(Effect.as({})), preview_type: (input) => invokeTargeted("type", input, input.timeoutMs).pipe(Effect.as({})), - preview_press: (input) => invokeTargeted("press", input).pipe(Effect.as({})), - preview_scroll: (input) => invokeTargeted("scroll", input).pipe(Effect.as({})), + preview_press: (input) => + invokeTargeted("press", input, input.timeoutMs).pipe(Effect.as({})), + preview_scroll: (input) => + invokeTargeted("scroll", input, input.timeoutMs).pipe(Effect.as({})), preview_evaluate: (input) => invokeTargeted("evaluate", input).pipe(Effect.map((result) => result ?? null)), preview_wait_for: (input) => invokeTargeted("waitFor", input, input.timeoutMs).pipe(Effect.as({})), preview_recording_start: (input) => - invokeTargeted("recordingStart", input ?? {}), + invokeTargeted( + "recordingStart", + input ?? {}, + input?.timeoutMs, + ), preview_recording_stop: (input) => invokeTargeted("recordingStop", input ?? {}), } satisfies Parameters[0]; diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 5ba688357b3..3091331cae4 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -6,6 +6,7 @@ import { PreviewAutomationOpenInput, PreviewAutomationPressInput, PreviewAutomationRecordingArtifact, + PreviewAutomationRecordingStartInput, PreviewAutomationRecordingStatus, PreviewAutomationResizeInput, PreviewAutomationResizeResult, @@ -186,7 +187,7 @@ export const PreviewRecordingStartTool = safeBrowserTool( Tool.make("preview_recording_start", { description: "Start recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted.", - parameters: PreviewAutomationTabTargetInput, + parameters: PreviewAutomationRecordingStartInput, success: PreviewAutomationRecordingStatus, failure: PreviewAutomationError, dependencies, diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index b5ef77dd2d8..56445f4953a 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -221,6 +221,26 @@ describe("browser recording", () => { expect(events.at(-1)).toBe("clear"); }); + it("bounds screencast and first-frame startup with the caller's remaining deadline", async () => { + vi.useFakeTimers(); + startScreencast.mockImplementationOnce(async () => { + events.push("start-screencast"); + }); + + const startPromise = startBrowserRecording("recording-tab", null, "recording-tab", 40); + const rejection = expect(startPromise).rejects.toMatchObject({ + operation: "wait-first-frame", + tabId: "recording-tab", + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(40); + + await rejection; + expect(startScreencast).toHaveBeenCalledWith("recording-tab", 40); + expect(stopScreencast).toHaveBeenCalledWith("recording-tab"); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set()); + }); + it("fixes hidden recording dimensions before MediaRecorder starts", async () => { const drawImage = vi.fn(); const fillRect = vi.fn(); diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 69297cfdbb8..cece5fd9870 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -272,13 +272,19 @@ const recordingStartupCancelledError = ( const isRecordingStarting = (recording: ActiveRecording): boolean => activeRecordings.get(recording.tabId) === recording && recording.lifecycle.phase === "starting"; -const waitForFirstFrameSize = async (recording: ActiveRecording): Promise => { +const waitForFirstFrameSize = async ( + recording: ActiveRecording, + timeoutMs = BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS, +): Promise => { if (recording.frameSizeEstablished) return true; let timeout: ReturnType | null = null; const outcome = await Promise.race([ recording.firstFrameSize, new Promise<"timeout">((resolve) => { - timeout = setTimeout(() => resolve("timeout"), BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS); + timeout = setTimeout( + () => resolve("timeout"), + Math.min(BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS, Math.max(0, timeoutMs)), + ); }), ]); if (timeout !== null) clearTimeout(timeout); @@ -314,9 +320,13 @@ export async function startBrowserRecording( tabId: string, threadRef: ScopedThreadRef | null = null, serverTabId = tabId, + timeoutMs?: number, ): Promise { const bridge = previewBridge; if (!bridge) throw new BrowserRecordingUnavailableError({ tabId }); + const deadline = timeoutMs === undefined ? null : Date.now() + timeoutMs; + const remainingStartupBudget = (): number | undefined => + deadline === null ? undefined : Math.max(0, deadline - Date.now()); const activeRecording = activeRecordings.get(tabId); if (activeRecording) { if (activeRecording.lifecycle.phase === "recording") { @@ -389,7 +399,9 @@ export async function startBrowserRecording( }); } try { - await bridge.recording.startScreencast(tabId); + const startBudget = remainingStartupBudget(); + if (startBudget === undefined) await bridge.recording.startScreencast(tabId); + else await bridge.recording.startScreencast(tabId, startBudget); } catch (cause) { if (!isRecordingStarting(recording)) { throw recordingStartupCancelledError(recording, cause); @@ -420,10 +432,17 @@ export async function startBrowserRecording( throw recordingStartupCancelledError(recording); }; await throwIfStartupCancelled(); - const hasFirstFrame = await waitForFirstFrameSize(recording); + const hasFirstFrame = await waitForFirstFrameSize( + recording, + remainingStartupBudget() ?? BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS, + ); await throwIfStartupCancelled(); - if (!hasFirstFrame) { - const cause = new Error(`No valid recording frame arrived for tab ${tabId}.`); + if (!hasFirstFrame || remainingStartupBudget() === 0) { + const cause = new Error( + hasFirstFrame + ? `Browser recording startup exceeded its deadline for tab ${tabId}.` + : `No valid recording frame arrived for tab ${tabId}.`, + ); const cleanupCause = await cleanupFailedRecordingStart(bridge, recording); throw new BrowserRecordingOperationError({ operation: "wait-first-frame", diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index d844b5314e5..f078eec73d6 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -588,7 +588,11 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) case "setColorScheme": { const ready = await requireReadyTab(); const input = request.input as PreviewAutomationSetColorSchemeInput; - await ready.bridge.setColorScheme(ready.runtimeTabId, input.colorScheme); + await ready.bridge.setColorScheme( + ready.runtimeTabId, + input.colorScheme, + remainingOperationBudget(input.timeoutMs ?? request.timeoutMs), + ); return { tabId: ready.tabId, colorScheme: input.colorScheme, @@ -637,16 +641,26 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "press": { const ready = await requireReadyTab(); + const input = request.input as Parameters[1]; return await ready.bridge.automation.press( ready.runtimeTabId, - request.input as Parameters[1], + previewAutomationInputWithRemainingTimeout( + input, + request.timeoutMs, + remainingOperationBudget, + ), ); } case "scroll": { const ready = await requireReadyTab(); + const input = request.input as Parameters[1]; return await ready.bridge.automation.scroll( ready.runtimeTabId, - request.input as Parameters[1], + previewAutomationInputWithRemainingTimeout( + input, + request.timeoutMs, + remainingOperationBudget, + ), ); } case "evaluate": { @@ -674,6 +688,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) ready.runtimeTabId, threadRef, ready.tabId, + remainingOperationBudget(request.timeoutMs), ); return { tabId: ready.tabId, diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 68cefecd449..268bd2c9601 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1000,6 +1000,16 @@ export const DesktopPreviewConfigInputSchema = Schema.Struct({ export const DesktopPreviewSetColorSchemeInputSchema = Schema.Struct({ tabId: DesktopPreviewTabIdSchema, colorScheme: DesktopPreviewColorSchemeSchema, + timeoutMs: Schema.optional( + Schema.Int.check(Schema.isGreaterThan(0)).check(Schema.isLessThanOrEqualTo(60_000)), + ), +}); + +export const DesktopPreviewRecordingStartInputSchema = Schema.Struct({ + tabId: DesktopPreviewTabIdSchema, + timeoutMs: Schema.optional( + Schema.Int.check(Schema.isGreaterThan(0)).check(Schema.isLessThanOrEqualTo(60_000)), + ), }); export const DesktopPreviewAnnotationThemeInputSchema = Schema.Struct({ @@ -1152,7 +1162,11 @@ export interface DesktopPreviewBridge { * Emulate `prefers-color-scheme` on the guest page ("system" clears the * override). Persists per tab and is re-applied across webview swaps. */ - setColorScheme: (tabId: string, colorScheme: DesktopPreviewColorScheme) => Promise; + setColorScheme: ( + tabId: string, + colorScheme: DesktopPreviewColorScheme, + timeoutMs?: number, + ) => Promise; /** Open the guest webview's DevTools (detached). */ openDevTools: (tabId: string) => Promise; /** Drop cookies + storage data for the preview partition (all tabs). */ @@ -1184,7 +1198,7 @@ export interface DesktopPreviewBridge { close: (tabId: string) => Promise; }; recording: { - startScreencast: (tabId: string) => Promise; + startScreencast: (tabId: string, timeoutMs?: number) => Promise; stopScreencast: (tabId: string) => Promise; save: ( tabId: string, diff --git a/packages/contracts/src/preview.test.ts b/packages/contracts/src/preview.test.ts index 24f429745ef..ec954548f76 100644 --- a/packages/contracts/src/preview.test.ts +++ b/packages/contracts/src/preview.test.ts @@ -15,8 +15,12 @@ import { PreviewAutomationHost, PreviewAutomationError, PreviewAutomationOpenInput, + PreviewAutomationPressInput, + PreviewAutomationRecordingStartInput, PreviewAutomationResizeInput, PreviewAutomationResizeResult, + PreviewAutomationScrollInput, + PreviewAutomationSetColorSchemeInput, PreviewAutomationStatus, } from "./previewAutomation.ts"; @@ -32,6 +36,21 @@ const decodeResizeResult = Schema.decodeUnknownSync(PreviewAutomationResizeResul const decodeAutomationHost = Schema.decodeUnknownSync(PreviewAutomationHost); const decodeAutomationError = Schema.decodeUnknownSync(PreviewAutomationError); const decodeAutomationStatus = Schema.decodeUnknownSync(PreviewAutomationStatus); +const decodePressInput = Schema.decodeUnknownSync(PreviewAutomationPressInput); +const decodeScrollInput = Schema.decodeUnknownSync(PreviewAutomationScrollInput); +const decodeSetColorSchemeInput = Schema.decodeUnknownSync(PreviewAutomationSetColorSchemeInput); +const decodeRecordingStartInput = Schema.decodeUnknownSync(PreviewAutomationRecordingStartInput); + +describe("Preview automation mutation deadlines", () => { + it("preserves explicit deadlines for mutations that follow overlay readiness", () => { + expect(decodePressInput({ key: "Enter", timeoutMs: 1_250 }).timeoutMs).toBe(1_250); + expect(decodeScrollInput({ deltaY: 100, timeoutMs: 1_250 }).timeoutMs).toBe(1_250); + expect(decodeSetColorSchemeInput({ colorScheme: "dark", timeoutMs: 1_250 }).timeoutMs).toBe( + 1_250, + ); + expect(decodeRecordingStartInput({ timeoutMs: 1_250 }).timeoutMs).toBe(1_250); + }); +}); describe("PreviewAutomationOpenInput", () => { it("accepts the inline preview visibility flag", () => { diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 733d512aec8..15442943ec3 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -267,6 +267,7 @@ export const PreviewAutomationSetColorSchemeInput = Schema.Struct({ description: "Emulated prefers-color-scheme for the page: light, dark, or system to follow the OS appearance.", }), + timeoutMs: OptionalTimeoutMs, }).annotate({ description: "Emulates prefers-color-scheme in the active browser tab without changing the OS or app theme.", @@ -377,6 +378,7 @@ export const PreviewAutomationPressInput = Schema.Struct({ description: "Modifier keys held while pressing key.", }), ), + timeoutMs: OptionalTimeoutMs, }).annotate({ description: "Presses one keyboard key in the active browser tab." }); export type PreviewAutomationPressInput = typeof PreviewAutomationPressInput.Type; @@ -398,6 +400,7 @@ export const PreviewAutomationScrollInput = Schema.Struct({ locator: Schema.optional(Locator).annotate({ description: "Playwright selector for a scrollable container. Omit to scroll the viewport.", }), + timeoutMs: OptionalTimeoutMs, }) .check( Schema.makeFilter((input) => { @@ -415,6 +418,12 @@ export const PreviewAutomationScrollInput = Schema.Struct({ }); export type PreviewAutomationScrollInput = typeof PreviewAutomationScrollInput.Type; +export const PreviewAutomationRecordingStartInput = Schema.Struct({ + ...PreviewAutomationTabTargetFields, + timeoutMs: OptionalTimeoutMs, +}).annotate({ description: "Starts recording the active browser tab." }); +export type PreviewAutomationRecordingStartInput = typeof PreviewAutomationRecordingStartInput.Type; + export const PreviewAutomationEvaluateInput = Schema.Struct({ ...PreviewAutomationTabTargetFields, expression: Schema.String.check(Schema.isTrimmed()) From c87599e68d28c511f9c3be185d5936f591fbf4b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Tue, 18 Aug 2026 01:12:09 +0100 Subject: [PATCH 41/43] fix(preview): bound recording finalization --- BRANCH_DETAILS.md | 6 +- apps/desktop/src/ipc/methods/preview.ts | 24 ++-- apps/desktop/src/preload.ts | 7 +- apps/desktop/src/preview/Manager.test.ts | 73 ++++++++++++ apps/desktop/src/preview/Manager.ts | 41 ++++++- apps/web/src/browser/browserRecording.test.ts | 29 +++++ apps/web/src/browser/browserRecording.ts | 108 +++++++++++++++--- .../preview/PreviewAutomationHosts.tsx | 17 ++- packages/contracts/src/ipc.test.ts | 21 ++++ packages/contracts/src/ipc.ts | 8 +- 10 files changed, 301 insertions(+), 33 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 67b0fb0b25d..1ebe84f6f4e 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -20,7 +20,7 @@ This branch does not add a second recording/PiP capture lifecycle or another hid Expected behavior: - Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout without making the execution budget shrink when the caller increases a short timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session while still holding an acquired control permit when a CDP command may be pending. Session removal and debugger teardown are atomic with respect to new session acquisition and bound to the exact acquired session, so late interruption or snapshot cleanup cannot detach a healthy replacement. Operations already queued on the retired semaphore detect that stale session and retry against its replacement. A request that times out while queued behind another action does not detach that action's shared debugger session. -- Click, type, press, scroll, and wait operations clamp their caller-supplied timeout to the remaining renderer host budget before entering the desktop control-session boundary. Color-scheme changes and recording startup likewise receive the remaining deadline after overlay readiness; a timed-out color-scheme command does not persist a late preference, and timed-out recording startup tears down its frame-capture session and renderer recording state. Operations without a caller timeout use the remaining bounded request budget rather than restarting the desktop default after renderer readiness work. +- Click, type, press, scroll, and wait operations clamp their caller-supplied timeout to the remaining renderer host budget before entering the desktop control-session boundary. Color-scheme changes and recording startup likewise receive the remaining deadline after overlay readiness; a timed-out color-scheme command does not persist a late preference, and timed-out recording startup tears down its frame-capture session and renderer recording state. Timed appearance persistence re-reads current tab state after CDP settles instead of relying on a pre-await snapshot. Recording stop bounds desktop capture shutdown, MediaRecorder settlement, blob conversion, and artifact persistence to the remaining deadline; a deadline failure retains captured chunks and the recording slot so finalization can be retried instead of silently losing the artifact. Operations without a caller timeout use the remaining bounded request budget rather than restarting the desktop default after renderer readiness work. - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Primary and fallback screenshot waits are clamped to the remaining control-session deadline, with budget reserved for fallback and result settlement, so a tight caller deadline can still return semantic data instead of being preempted by the outer session timeout. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, an actually timed-out CDP capture resets the session before releasing its control permit, queued work reattaches before issuing its first command, and a capture skipped before CDP runs leaves the healthy session attached. The semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. `apps/web/src/browser/desktopTabLifetime.ts` passes the upstream browser appearance default through `DesktopPreviewCreateTabInputSchema` in `packages/contracts/src/ipc.ts`; `apps/desktop/src/preview/Manager.ts` normalizes that value. A non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path, while tabs following the system scheme stay detached until the next automation operation. - Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the server epoch replaces the runtime guest while staging is pending, the snapshot fails immediately with `PreviewAutomationTargetUnavailableError` instead of waiting on the stale staging marker. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. @@ -30,7 +30,7 @@ Expected behavior: - `PreviewAutomationHosts.tsx` resolves `browserDefaults.ts` once at the start of each automation-open request, before taking the session snapshot that pins a reused runtime guest. That single settings snapshot supplies the new-tab viewport and the automatic floating-preview preference, so creation and presentation cannot observe different settings during one request, while cold settings hydration cannot mix a pre-await session snapshot with a post-await server epoch. Explicit `open` or its deprecated `show` alias remains authoritative; when both are omitted, `autoShowFloatingPreview` decides presentation. The resulting `shouldPresentPreview` value is passed unchanged to `previewAutomationOpenReadiness.ts`, so a reused rendered tab left in the background does not wait for visibility while an explicitly shown tab does. After settings and session synchronization, the host rechecks its remaining deadline immediately before tab creation and every later irreversible open-side mutation, so an already expired request cannot create, resize, reveal, or navigate a preview. A newly created tab applies its server snapshot and assigned tab id, uses the configured viewport or the branch's deterministic 1280×800 fallback when the snapshot remains `fill`, initiates any requested selection, and acknowledges server-side creation without depending on cold React panel rendering, Electron overlay registration, or page readiness. Its initial URL continues loading exactly once in that same tab; later wait, snapshot, or interaction operations own attachment and page readiness. Reopening an existing shown tab selects both the preview-state tab and its matching inline mini-player surface, then waits for stable presentation and reasserts that selection across same-server route hydration or session reconciliation. A server-epoch change aborts the pending open with `PreviewAutomationTargetUnavailableError`; the old request never adopts the replacement runtime guest. - The standard dev runner keeps local navigation and direct backend URLs on `127.0.0.1`. Browser modes use a single origin: they leave client HTTP/WebSocket URLs and generic `HOST` unset so remote sharing and origin-derived HMR keep working, while Vite's default listener and its default backend proxy use explicit IPv4 loopback. Explicit IPv6 backend binds proxy through IPv6 loopback. Desktop mode pins `HOST` and its renderer/backend URLs to `127.0.0.1`; server-only mode keeps direct HTTP/WebSocket URLs on the same IPv4 loopback. - Retained browser guests subscribe only to render state for their own epoch-scoped runtime tab, plus their thread's active panel and mini-player selection. Presentation or background-capture updates for another tab do not rerender every mounted `HostedBrowserWebview`; selection changes override a stale surface-visible flag so background staging remains nearly transparent and exposes its readiness marker, and staging derives its viewport-fitted rectangle from the target tab's own stable rectangle. Only the active guest is exposed through the host accessibility tree; a nearly transparent background-capture guest stays `aria-hidden` without affecting CDP accessibility-tree collection against the guest target. -- Rendered viewport readiness revalidates the epoch-scoped runtime guest after each awaited guest measurement and clamps every polling sleep to the remaining resize deadline, preserving typed target-replacement and viewport-timeout failures for short budgets. +- Rendered viewport readiness revalidates the epoch-scoped runtime guest after each awaited guest measurement and clamps every polling sleep to the remaining resize deadline, preserving typed target-replacement and viewport-timeout failures for short budgets. The serialized viewport mutation also rechecks that deadline after acquiring its mutation queue, before sending the server resize. - The primary pairing route watches for later URL-fragment changes while it remains mounted. Navigating an already-loaded `/pair` document to `/pair#token=...` claims each new token once, removes the secret fragment, and runs the normal pairing exchange without requiring a reload or a second desktop window. Multiple tokens received while an exchange is pending are serialized, and the submitting state remains active until every queued exchange settles. - `dev:desktop` derives `T3CODE_DESKTOP_USER_DATA_DIR=/userdata/electron` whenever the runner has an explicit base directory. Desktop configuration resolves that override to an absolute path, and app identity uses it before legacy migration or the normal Electron user-data default. This keeps an isolated worktree dev desktop from reusing an installed or earlier development profile whose incompatible IndexedDB schema can prevent the renderer from starting. Packaged/default startup remains unchanged when no override is supplied. @@ -76,7 +76,7 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.t Current verification: -- The branch-focused suite passed 278 tests across 19 files on Windows, including the new bounded post-overlay mutations, recording-start cleanup, short-deadline viewport polling, post-read runtime-replacement, and background-guest accessibility regressions. The two unchanged desktop path-fixture files in the command above currently produce eight POSIX-versus-Windows path assertion failures; the affected runtime behavior passes in the remaining focused suite. +- The branch-focused suite passed 281 tests across 19 files on Windows, including bounded post-overlay and queued resize mutations, retryable recording finalization, recording-start cleanup, current-state appearance persistence, short-deadline viewport polling, post-read runtime-replacement, and background-guest accessibility regressions. The two unchanged desktop path-fixture files in the command above currently produce eight POSIX-versus-Windows path assertion failures; the affected runtime behavior passes in the remaining focused suite. - The incoming browser-default, provider-access, pull-request budget, tooltip, update-copy, and settings coverage passed 552 tests across 26 files. The focused open-policy and open-session subset passes 18 tests, including a shared explicit-over-default presentation decision and a reused rendered tab that remains in the background without a visibility wait. - Desktop, server, contracts, and mobile typechecks pass. Web typecheck currently reports 16 existing errors in `BranchToolbarBranchSelector.tsx`, `ModelPickerContent.tsx`, `PreviewAutomationHosts.tsx` at the pre-existing registry access, `FontFamilyPicker.tsx`, `use-atom-command.ts`, and `use-atom-query-runner.ts`; none are in the browser-default integration changed here. - An isolated web client on ports `5744`/`13784` paired and loaded successfully. Settings → Integrations exposed the browser defaults and agent-access setting; agent browser access could be disabled and restored. The non-Electron client correctly disabled desktop-only viewport, zoom, and appearance controls. diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 0fb40ea128f..3d81e7a9843 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -12,6 +12,7 @@ import { DesktopPreviewNavigateInputSchema, DesktopPreviewRecordingArtifactSchema, DesktopPreviewRecordingStartInputSchema, + DesktopPreviewRecordingStopInputSchema, DesktopPreviewRecordingSaveInputSchema, DesktopPreviewRegisterWebviewInputSchema, DesktopPreviewScreenshotArtifactSchema, @@ -178,11 +179,15 @@ export const startRecording = DesktopIpc.makeIpcMethod({ yield* manager.startRecording(tabId, timeoutMs); }), }); -export const stopRecording = tabMethod( - IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL, - "desktop.ipc.preview.stopRecording", - (manager, tabId) => manager.stopRecording(tabId), -); +export const stopRecording = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL, + payload: DesktopPreviewRecordingStopInputSchema, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.preview.stopRecording")(function* ({ tabId, timeoutMs }) { + const manager = yield* PreviewManager.PreviewManager; + yield* manager.stopRecording(tabId, timeoutMs); + }), +}); export const openPictureInPicture = tabMethod( IpcChannels.PREVIEW_PICTURE_IN_PICTURE_OPEN_CHANNEL, "desktop.ipc.preview.openPictureInPicture", @@ -367,9 +372,14 @@ export const saveRecording = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PREVIEW_RECORDING_SAVE_CHANNEL, payload: DesktopPreviewRecordingSaveInputSchema, result: DesktopPreviewRecordingArtifactSchema, - handler: Effect.fn("desktop.ipc.preview.saveRecording")(function* ({ tabId, mimeType, data }) { + handler: Effect.fn("desktop.ipc.preview.saveRecording")(function* ({ + tabId, + mimeType, + data, + timeoutMs, + }) { const manager = yield* PreviewManager.PreviewManager; - return yield* manager.saveRecording(tabId, mimeType, data); + return yield* manager.saveRecording(tabId, mimeType, data, timeoutMs); }), }); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index b11ed469f89..36f1cb8b99a 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -213,13 +213,14 @@ contextBridge.exposeInMainWorld("desktopBridge", { recording: { startScreencast: (tabId, timeoutMs) => ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_START_CHANNEL, { tabId, timeoutMs }), - stopScreencast: (tabId) => - ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL, { tabId }), - save: (tabId, mimeType, data) => + stopScreencast: (tabId, timeoutMs) => + ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL, { tabId, timeoutMs }), + save: (tabId, mimeType, data, timeoutMs) => ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_SAVE_CHANNEL, { tabId, mimeType, data, + timeoutMs, }), onFrame: (listener) => { const wrappedListener = (_event: Electron.IpcRendererEvent, frame: unknown) => { diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index e7e9dc4534b..39c48a26d3a 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1788,6 +1788,79 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("re-reads color-scheme state after a bounded CDP mutation settles", () => + withManager((manager) => + Effect.gen(function* () { + let attached = false; + let darkCommands = 0; + let finishBoundedDark: (() => void) | undefined; + const sendCommand = vi.fn( + async (method: string, parameters?: { features?: ReadonlyArray<{ value: string }> }) => { + if (method !== "Emulation.setEmulatedMedia") return undefined; + if (parameters?.features?.[0]?.value !== "dark") return undefined; + darkCommands += 1; + if (darkCommands !== 2) return undefined; + await new Promise((resolve) => { + finishBoundedDark = resolve; + }); + return undefined; + }, + ); + fromId.mockReturnValue({ + id: 42, + isDestroyed: () => false, + isDevToolsOpened: () => false, + getType: () => "webview", + getURL: () => "https://example.com", + getTitle: () => "Example", + isLoading: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => attached, + attach: vi.fn(() => { + attached = true; + }), + detach: vi.fn(() => { + attached = false; + }), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + } as never); + const states: PreviewManager.PreviewTabState[] = []; + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + states.push(state); + }), + ); + yield* manager.createTab("tab_scheme_reread"); + yield* manager.registerWebview("tab_scheme_reread", 42); + yield* Effect.yieldNow; + yield* manager.setColorScheme("tab_scheme_reread", "dark"); + + const boundedDark = yield* manager + .setColorScheme("tab_scheme_reread", "dark", 1_000) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + yield* manager.setColorScheme("tab_scheme_reread", "light"); + expect(states.at(-1)?.colorScheme).toBe("light"); + + finishBoundedDark?.(); + yield* Fiber.join(boundedDark); + + expect(states.at(-1)?.colorScheme).toBe("dark"); + }), + ), + ); + effectIt.effect("blocks late webview and capture starts during tab close", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 1b8b91606b5..6592b60b4b3 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2354,7 +2354,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }), timeoutMs, ); - if (tab.colorScheme !== colorScheme) { + const currentTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if (!currentTab) { + return yield* new PreviewTabNotFoundError({ tabId }); + } + if (currentTab.colorScheme !== colorScheme) { yield* update(tabId, { colorScheme }); } return; @@ -2918,11 +2922,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return yield* new PreviewAutomationTimeoutError({ tabId, timeoutMs }); }); - const stopRecording = Effect.fn("PreviewManager.stopRecording")(function* (tabId: string) { - yield* stopFrameCapture(tabId, "recording"); + const stopRecording = Effect.fn("PreviewManager.stopRecording")(function* ( + tabId: string, + timeoutMs?: number, + ) { + const stop = stopFrameCapture(tabId, "recording"); + if (timeoutMs === undefined) { + yield* stop; + return; + } + const result = yield* stop.pipe(Effect.timeoutOption(automationExecutionBudget(timeoutMs))); + if (Option.isSome(result)) return; + return yield* new PreviewAutomationTimeoutError({ tabId, timeoutMs }); }); - const saveRecording = Effect.fn("PreviewManager.saveRecording")(function* ( + const performSaveRecording = Effect.fn("PreviewManager.performSaveRecording")(function* ( tabId: string, mimeType: string, data: Uint8Array, @@ -2963,6 +2977,19 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }; }); + const saveRecording = Effect.fn("PreviewManager.saveRecording")(function* ( + tabId: string, + mimeType: string, + data: Uint8Array, + timeoutMs?: number, + ) { + const save = performSaveRecording(tabId, mimeType, data); + if (timeoutMs === undefined) return yield* save; + const result = yield* save.pipe(Effect.timeoutOption(automationExecutionBudget(timeoutMs))); + if (Option.isSome(result)) return result.value; + return yield* new PreviewAutomationTimeoutError({ tabId, timeoutMs }); + }); + const automationStatus = Effect.fn("PreviewManager.automationStatus")(function* (tabId: string) { const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); if (!tab || tab.webContentsId == null) { @@ -4325,11 +4352,15 @@ export class PreviewManager extends Context.Service< tabId: string, timeoutMs?: number, ) => Effect.Effect; - readonly stopRecording: (tabId: string) => Effect.Effect; + readonly stopRecording: ( + tabId: string, + timeoutMs?: number, + ) => Effect.Effect; readonly saveRecording: ( tabId: string, mimeType: string, data: Uint8Array, + timeoutMs?: number, ) => Effect.Effect; readonly automationStatus: ( tabId: string, diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index 56445f4953a..0edcd943fe2 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -497,6 +497,35 @@ describe("browser recording", () => { expect(save).toHaveBeenCalledOnce(); }); + it("retains captured chunks when a bounded stop expires so finalization can be retried", async () => { + vi.useFakeTimers(); + let finishStoppingScreencast: (() => void) | undefined; + stopScreencast.mockImplementationOnce(async () => { + await new Promise((resolve) => { + finishStoppingScreencast = resolve; + }); + }); + await startBrowserRecording("recording-tab"); + + const stopPromise = stopBrowserRecording("recording-tab", 40); + const rejection = expect(stopPromise).rejects.toMatchObject({ + operation: "stop-deadline", + tabId: "recording-tab", + }); + await vi.advanceTimersByTimeAsync(40); + + await rejection; + expect(stopScreencast).toHaveBeenCalledWith("recording-tab", 40); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set(["recording-tab"])); + + finishStoppingScreencast?.(); + await Promise.resolve(); + await expect(stopBrowserRecording("recording-tab")).resolves.toMatchObject({ + tabId: "recording-tab", + }); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set()); + }); + it("finishes startup before stopping so an active recording yields an artifact", async () => { let finishStartingScreencast: (() => void) | undefined; startScreencast.mockImplementationOnce(async () => { diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index cece5fd9870..2153a3ee092 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -59,6 +59,7 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass isBrowserRecordingOperationError(error) && error.operation === "wait-startup"; +const recordingStopDeadlineError = (tabId: string): BrowserRecordingOperationError => + new BrowserRecordingOperationError({ + operation: "stop-deadline", + tabId, + cause: new Error(`Browser recording stop exceeded its deadline for tab ${tabId}.`), + }); + +export const isBrowserRecordingStopDeadlineError = ( + error: unknown, +): error is BrowserRecordingOperationError => + isBrowserRecordingOperationError(error) && error.operation === "stop-deadline"; + +const remainingRecordingStopBudget = ( + deadline: number | null, + tabId: string, +): number | undefined => { + if (deadline === null) return undefined; + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) throw recordingStopDeadlineError(tabId); + return remainingMs; +}; + +const awaitWithinRecordingStopDeadline = async ( + promise: Promise, + deadline: number | null, + tabId: string, +): Promise => { + const remainingMs = remainingRecordingStopBudget(deadline, tabId); + if (remainingMs === undefined) return await promise; + let timeout: ReturnType | null = null; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timeout = setTimeout(() => reject(recordingStopDeadlineError(tabId)), remainingMs); + }), + ]); + } finally { + if (timeout !== null) clearTimeout(timeout); + } +}; + export async function startBrowserRecording( tabId: string, threadRef: ScopedThreadRef | null = null, @@ -518,6 +561,7 @@ export async function startBrowserRecording( const finalizeBrowserRecording = async ( bridge: NonNullable, recording: ActiveRecording, + deadline: number | null, ): Promise => { const { tabId } = recording; let result: @@ -527,10 +571,22 @@ const finalizeBrowserRecording = async ( } | { readonly _tag: "Failure"; readonly error: unknown }; try { - await waitForRecordingStartupToSettle(recording); + await awaitWithinRecordingStopDeadline( + waitForRecordingStartupToSettle(recording), + deadline, + tabId, + ); try { - await bridge.recording.stopScreencast(tabId); + const stopBudget = remainingRecordingStopBudget(deadline, tabId); + await awaitWithinRecordingStopDeadline( + stopBudget === undefined + ? bridge.recording.stopScreencast(tabId) + : bridge.recording.stopScreencast(tabId, stopBudget), + deadline, + tabId, + ); } catch (cause) { + if (isBrowserRecordingStopDeadlineError(cause)) throw cause; throw new BrowserRecordingOperationError({ operation: "stop-screencast", tabId, @@ -541,8 +597,13 @@ const finalizeBrowserRecording = async ( result = { _tag: "Success", artifact: null }; } else { try { - await stopMediaRecorder(recording.recorder); + await awaitWithinRecordingStopDeadline( + stopMediaRecorder(recording.recorder), + deadline, + tabId, + ); } catch (cause) { + if (isBrowserRecordingStopDeadlineError(cause)) throw cause; throw new BrowserRecordingOperationError({ operation: "stop-media-recorder", tabId, @@ -551,13 +612,20 @@ const finalizeBrowserRecording = async ( } try { const blob = new Blob(recording.chunks, { type: recording.mimeType }); - const artifact = await bridge.recording.save( + const data = new Uint8Array( + await awaitWithinRecordingStopDeadline(blob.arrayBuffer(), deadline, tabId), + ); + const saveBudget = remainingRecordingStopBudget(deadline, tabId); + const artifact = await awaitWithinRecordingStopDeadline( + saveBudget === undefined + ? bridge.recording.save(tabId, recording.mimeType, data) + : bridge.recording.save(tabId, recording.mimeType, data, saveBudget), + deadline, tabId, - recording.mimeType, - new Uint8Array(await blob.arrayBuffer()), ); result = { _tag: "Success", artifact }; } catch (cause) { + if (isBrowserRecordingStopDeadlineError(cause)) throw cause; throw new BrowserRecordingOperationError({ operation: "save-artifact", tabId, @@ -569,11 +637,13 @@ const finalizeBrowserRecording = async ( result = { _tag: "Failure", error }; } - if (result._tag === "Failure" && isStartupWaitTimeout(result.error)) { - // Do not clear `active` yet. The renderer-side start promise can still - // resolve later, and its cancellation path will call `stopScreencast`. - // Keeping the slot reserved prevents a newer recording for this tab from - // being started and then accidentally stopped by the older late cleanup. + if ( + result._tag === "Failure" && + (isStartupWaitTimeout(result.error) || isBrowserRecordingStopDeadlineError(result.error)) + ) { + // Keep the slot and captured chunks available. Startup may still need its + // cancellation cleanup, while a deadline failure can be retried to finish + // saving an already-stopped MediaRecorder without losing its artifact. throw result.error; } @@ -623,15 +693,27 @@ const discardBrowserRecording = async ( export function stopBrowserRecording( tabId: string, + timeoutMs?: number, ): Promise { const bridge = previewBridge; const recording = activeRecordings.get(tabId); if (!bridge || !recording) return Promise.resolve(null); - if (recording.lifecycle.phase === "stopping") return recording.lifecycle.stopPromise; + const deadline = timeoutMs === undefined ? null : Date.now() + timeoutMs; + if (recording.lifecycle.phase === "stopping") { + return awaitWithinRecordingStopDeadline(recording.lifecycle.stopPromise, deadline, tabId); + } const stopPromise = Promise.resolve() - .then(() => finalizeBrowserRecording(bridge, recording)) + .then(() => finalizeBrowserRecording(bridge, recording, deadline)) .catch((error) => { + if ( + isBrowserRecordingStopDeadlineError(error) && + activeRecordings.get(recording.tabId) === recording && + recording.lifecycle.phase === "stopping" && + recording.lifecycle.stopPromise === stopPromise + ) { + recording.lifecycle = { phase: "recording" }; + } if (isStartupWaitTimeout(error) && activeRecordings.get(recording.tabId) === recording) { const cleanupAfterStartup = recording.startupSettled.then(() => discardBrowserRecording(bridge, recording), diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index f078eec73d6..7e1dff8ee73 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -31,6 +31,7 @@ import { } from "~/previewStateStore"; import { resolveBrowserNavigationTarget } from "~/browser/browserTargetResolver"; import { + isBrowserRecordingStopDeadlineError, readActiveBrowserRecordingTargets, startBrowserRecording, stopBrowserRecording, @@ -510,6 +511,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) const input = request.input as PreviewAutomationResizeInput; const setting = resolvePreviewViewport(input); const applied = await runBrowserViewportMutation(ready.runtimeTabId, async () => { + remainingOperationBudget(input.timeoutMs ?? request.timeoutMs); const operationState = assertPreviewRuntimeCurrent( threadRef, ready.tabId, @@ -710,7 +712,20 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) const stopRuntimeTabId = activeRecordings.find((recording) => recording.serverTabId === stopTabId) ?.runtimeTabId ?? null; - const artifact = stopRuntimeTabId ? await stopBrowserRecording(stopRuntimeTabId) : null; + let artifact = null; + if (stopRuntimeTabId) { + try { + artifact = await stopBrowserRecording( + stopRuntimeTabId, + remainingOperationBudget(request.timeoutMs), + ); + } catch (cause) { + if (isBrowserRecordingStopDeadlineError(cause)) { + remainingOperationBudget(request.timeoutMs); + } + throw cause; + } + } if (!artifact || !stopTabId) { return raisePreviewAutomationHostError( new PreviewAutomationRecordingNotActiveError({ diff --git a/packages/contracts/src/ipc.test.ts b/packages/contracts/src/ipc.test.ts index ffc4ece1cca..b0200bdcd55 100644 --- a/packages/contracts/src/ipc.test.ts +++ b/packages/contracts/src/ipc.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it } from "vite-plus/test"; import { DesktopEnvironmentBootstrapSchema, DesktopPreviewAutomationSnapshotInputSchema, + DesktopPreviewRecordingSaveInputSchema, + DesktopPreviewRecordingStopInputSchema, } from "./ipc.ts"; describe("DesktopEnvironmentBootstrapSchema", () => { @@ -40,6 +42,25 @@ describe("DesktopEnvironmentBootstrapSchema", () => { }); }); +describe("desktop recording finalization deadlines", () => { + it("preserves explicit stop and save timeouts", () => { + const decodeStop = Schema.decodeUnknownSync(DesktopPreviewRecordingStopInputSchema); + const decodeSave = Schema.decodeUnknownSync(DesktopPreviewRecordingSaveInputSchema); + const data = new Uint8Array([1, 2, 3]); + + expect(decodeStop({ tabId: "tab-1", timeoutMs: 1_250 })).toEqual({ + tabId: "tab-1", + timeoutMs: 1_250, + }); + expect(decodeSave({ tabId: "tab-1", mimeType: "video/webm", data, timeoutMs: 1_250 })).toEqual({ + tabId: "tab-1", + mimeType: "video/webm", + data, + timeoutMs: 1_250, + }); + }); +}); + describe("DesktopPreviewAutomationSnapshotInputSchema", () => { const decode = Schema.decodeUnknownSync(DesktopPreviewAutomationSnapshotInputSchema); diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 268bd2c9601..0da2c72c744 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1012,6 +1012,8 @@ export const DesktopPreviewRecordingStartInputSchema = Schema.Struct({ ), }); +export const DesktopPreviewRecordingStopInputSchema = DesktopPreviewRecordingStartInputSchema; + export const DesktopPreviewAnnotationThemeInputSchema = Schema.Struct({ theme: DesktopPreviewAnnotationThemeSchema, }); @@ -1024,6 +1026,9 @@ export const DesktopPreviewRecordingSaveInputSchema = Schema.Struct({ tabId: DesktopPreviewTabIdSchema, mimeType: Schema.String.check(Schema.isTrimmed()).check(Schema.isNonEmpty()), data: Schema.Uint8Array, + timeoutMs: Schema.optional( + Schema.Int.check(Schema.isGreaterThan(0)).check(Schema.isLessThanOrEqualTo(60_000)), + ), }); export const DesktopPreviewAutomationClickInputSchema = Schema.Struct({ @@ -1199,11 +1204,12 @@ export interface DesktopPreviewBridge { }; recording: { startScreencast: (tabId: string, timeoutMs?: number) => Promise; - stopScreencast: (tabId: string) => Promise; + stopScreencast: (tabId: string, timeoutMs?: number) => Promise; save: ( tabId: string, mimeType: string, data: Uint8Array, + timeoutMs?: number, ) => Promise; onFrame: (listener: (frame: DesktopPreviewRecordingFrame) => void) => () => void; }; From 6716ed7a2d3bf07f53d6e58f56bc9d3392a7f6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Tue, 18 Aug 2026 01:29:29 +0100 Subject: [PATCH 42/43] fix(preview): preserve recording retries --- BRANCH_DETAILS.md | 4 +- apps/desktop/src/preview/Manager.test.ts | 83 +++++++++++++++++++ apps/desktop/src/preview/Manager.ts | 62 +++++++++----- apps/web/src/browser/browserRecording.test.ts | 52 ++++++++++++ apps/web/src/browser/browserRecording.ts | 58 ++++++++++--- 5 files changed, 221 insertions(+), 38 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index 1ebe84f6f4e..c2decd3d735 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -20,7 +20,7 @@ This branch does not add a second recording/PiP capture lifecycle or another hid Expected behavior: - Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout without making the execution budget shrink when the caller increases a short timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session while still holding an acquired control permit when a CDP command may be pending. Session removal and debugger teardown are atomic with respect to new session acquisition and bound to the exact acquired session, so late interruption or snapshot cleanup cannot detach a healthy replacement. Operations already queued on the retired semaphore detect that stale session and retry against its replacement. A request that times out while queued behind another action does not detach that action's shared debugger session. -- Click, type, press, scroll, and wait operations clamp their caller-supplied timeout to the remaining renderer host budget before entering the desktop control-session boundary. Color-scheme changes and recording startup likewise receive the remaining deadline after overlay readiness; a timed-out color-scheme command does not persist a late preference, and timed-out recording startup tears down its frame-capture session and renderer recording state. Timed appearance persistence re-reads current tab state after CDP settles instead of relying on a pre-await snapshot. Recording stop bounds desktop capture shutdown, MediaRecorder settlement, blob conversion, and artifact persistence to the remaining deadline; a deadline failure retains captured chunks and the recording slot so finalization can be retried instead of silently losing the artifact. Operations without a caller timeout use the remaining bounded request budget rather than restarting the desktop default after renderer readiness work. +- Click, type, press, scroll, and wait operations clamp their caller-supplied timeout to the remaining renderer host budget before entering the desktop control-session boundary. Color-scheme changes and recording startup likewise receive the remaining deadline after overlay readiness; a timed-out color-scheme command does not persist a late preference, and timed-out recording startup tears down its frame-capture session and renderer recording state. Timed appearance persistence re-reads current tab state after CDP settles and reapplies to the current guest if the webview was replaced instead of trusting a pre-await target snapshot. Recording stop bounds desktop capture shutdown, MediaRecorder settlement, blob conversion, and artifact persistence to the remaining deadline; renderer and desktop-originated deadline failures retain captured chunks and the recording slot so finalization can be retried instead of silently losing the artifact. An in-flight artifact-save promise is shared with that retry, preventing duplicate timestamped files when the renderer deadline wins before desktop IPC settles. Operations without a caller timeout use the remaining bounded request budget rather than restarting the desktop default after renderer readiness work. - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Primary and fallback screenshot waits are clamped to the remaining control-session deadline, with budget reserved for fallback and result settlement, so a tight caller deadline can still return semantic data instead of being preempted by the outer session timeout. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, an actually timed-out CDP capture resets the session before releasing its control permit, queued work reattaches before issuing its first command, and a capture skipped before CDP runs leaves the healthy session attached. The semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. `apps/web/src/browser/desktopTabLifetime.ts` passes the upstream browser appearance default through `DesktopPreviewCreateTabInputSchema` in `packages/contracts/src/ipc.ts`; `apps/desktop/src/preview/Manager.ts` normalizes that value. A non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path, while tabs following the system scheme stay detached until the next automation operation. - Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the server epoch replaces the runtime guest while staging is pending, the snapshot fails immediately with `PreviewAutomationTargetUnavailableError` instead of waiting on the stale staging marker. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. @@ -76,7 +76,7 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.t Current verification: -- The branch-focused suite passed 281 tests across 19 files on Windows, including bounded post-overlay and queued resize mutations, retryable recording finalization, recording-start cleanup, current-state appearance persistence, short-deadline viewport polling, post-read runtime-replacement, and background-guest accessibility regressions. The two unchanged desktop path-fixture files in the command above currently produce eight POSIX-versus-Windows path assertion failures; the affected runtime behavior passes in the remaining focused suite. +- The branch-focused suite passed 284 tests across 19 files on Windows, including bounded post-overlay and queued resize mutations, retryable and deduplicated recording finalization across renderer and desktop timeouts, recording-start cleanup, replacement-guest appearance persistence, short-deadline viewport polling, post-read runtime-replacement, and background-guest accessibility regressions. The two unchanged desktop path-fixture files in the command above currently produce eight POSIX-versus-Windows path assertion failures; the affected runtime behavior passes in the remaining focused suite. - The incoming browser-default, provider-access, pull-request budget, tooltip, update-copy, and settings coverage passed 552 tests across 26 files. The focused open-policy and open-session subset passes 18 tests, including a shared explicit-over-default presentation decision and a reused rendered tab that remains in the background without a visibility wait. - Desktop, server, contracts, and mobile typechecks pass. Web typecheck currently reports 16 existing errors in `BranchToolbarBranchSelector.tsx`, `ModelPickerContent.tsx`, `PreviewAutomationHosts.tsx` at the pre-existing registry access, `FontFamilyPicker.tsx`, `use-atom-command.ts`, and `use-atom-query-runner.ts`; none are in the browser-default integration changed here. - An isolated web client on ports `5744`/`13784` paired and loaded successfully. Settings → Integrations exposed the browser defaults and agent-access setting; agent browser access could be disabled and restored. The non-Electron client correctly disabled desktop-only viewport, zoom, and appearance controls. diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 39c48a26d3a..42d328accb7 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1861,6 +1861,89 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("reapplies a bounded color scheme after the webview is replaced", () => + withManager((manager) => + Effect.gen(function* () { + let finishFirstGuest: (() => void) | undefined; + const firstSendCommand = vi.fn( + async (method: string, parameters?: { features?: ReadonlyArray<{ value: string }> }) => { + if ( + method === "Emulation.setEmulatedMedia" && + parameters?.features?.[0]?.value === "dark" + ) { + await new Promise((resolve) => { + finishFirstGuest = resolve; + }); + } + return undefined; + }, + ); + const replacementSendCommand = vi.fn(async () => undefined); + const makeWebContents = ( + id: number, + sendCommand: typeof firstSendCommand | typeof replacementSendCommand, + ) => { + let attached = false; + return { + id, + isDestroyed: () => false, + isDevToolsOpened: () => false, + getType: () => "webview", + getURL: () => "https://example.com", + getTitle: () => "Example", + isLoading: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => attached, + attach: vi.fn(() => { + attached = true; + }), + detach: vi.fn(() => { + attached = false; + }), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + } as never; + }; + const first = makeWebContents(42, firstSendCommand); + const replacement = makeWebContents(43, replacementSendCommand); + fromId.mockImplementation((id) => (id === 42 ? first : id === 43 ? replacement : null)); + const states: PreviewManager.PreviewTabState[] = []; + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + states.push(state); + }), + ); + yield* manager.createTab("tab_scheme_replacement"); + yield* manager.registerWebview("tab_scheme_replacement", 42); + yield* Effect.yieldNow; + + const mutation = yield* manager + .setColorScheme("tab_scheme_replacement", "dark", 1_000) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + yield* manager.registerWebview("tab_scheme_replacement", 43); + yield* Effect.yieldNow; + finishFirstGuest?.(); + yield* Fiber.join(mutation); + + expect(replacementSendCommand).toHaveBeenCalledWith("Emulation.setEmulatedMedia", { + features: [{ name: "prefers-color-scheme", value: "dark" }], + }); + expect(states.at(-1)?.colorScheme).toBe("dark"); + }), + ), + ); + effectIt.effect("blocks late webview and capture starts during tab close", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 6592b60b4b3..2fb05b39264 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2338,30 +2338,46 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return yield* new PreviewTabNotFoundError({ tabId }); } if (timeoutMs !== undefined) { - const wc = yield* requireWebContents(tabId); - yield* withControlSession( - tabId, - wc, - "set-color-scheme", - (send) => - send("Emulation.setEmulatedMedia", { - features: [ - { - name: "prefers-color-scheme", - value: colorScheme === "system" ? "" : colorScheme, - }, - ], - }), - timeoutMs, - ); - const currentTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); - if (!currentTab) { - return yield* new PreviewTabNotFoundError({ tabId }); - } - if (currentTab.colorScheme !== colorScheme) { - yield* update(tabId, { colorScheme }); + const deadline = (yield* currentMillis) + timeoutMs; + let target = yield* requireWebContents(tabId); + while (true) { + const remainingTimeoutMs = deadline - (yield* currentMillis); + if (remainingTimeoutMs <= 0) { + return yield* new PreviewAutomationTimeoutError({ tabId, timeoutMs }); + } + yield* withControlSession( + tabId, + target, + "set-color-scheme", + (send) => + send("Emulation.setEmulatedMedia", { + features: [ + { + name: "prefers-color-scheme", + value: colorScheme === "system" ? "" : colorScheme, + }, + ], + }), + remainingTimeoutMs, + ); + const currentTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if (!currentTab) { + return yield* new PreviewTabNotFoundError({ tabId }); + } + if (currentTab.webContentsId !== target.id) { + target = yield* requireWebContents(tabId); + continue; + } + if (currentTab.colorScheme !== colorScheme) { + yield* update(tabId, { colorScheme }); + } + const appliedTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if (!appliedTab) { + return yield* new PreviewTabNotFoundError({ tabId }); + } + if (appliedTab.webContentsId === target.id) return; + target = yield* requireWebContents(tabId); } - return; } if (tab.colorScheme !== colorScheme) { // Record the choice even when the CDP call below can't run yet (no diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index 0edcd943fe2..dc44da590a1 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -526,6 +526,58 @@ describe("browser recording", () => { expect(readActiveBrowserRecordingTabIds()).toEqual(new Set()); }); + it("shares an in-flight artifact save with a retry after the renderer deadline", async () => { + vi.useFakeTimers(); + let finishSaving: ((artifact: Awaited>) => void) | undefined; + save.mockImplementationOnce( + async () => + await new Promise>>((resolve) => { + finishSaving = resolve; + }), + ); + await startBrowserRecording("recording-tab"); + + const firstStop = stopBrowserRecording("recording-tab", 40); + const rejection = expect(firstStop).rejects.toMatchObject({ operation: "stop-deadline" }); + await vi.advanceTimersByTimeAsync(40); + await rejection; + + const retry = stopBrowserRecording("recording-tab"); + expect(save).toHaveBeenCalledOnce(); + finishSaving?.({ + id: "recording-test", + tabId: "recording-tab", + path: "/tmp/recording-test.webm", + mimeType: "video/webm", + sizeBytes: 0, + createdAt: "2026-06-26T00:00:00.000Z", + }); + + await expect(retry).resolves.toMatchObject({ id: "recording-test" }); + expect(save).toHaveBeenCalledOnce(); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set()); + }); + + it("keeps recording state retryable when the desktop stop deadline rejects first", async () => { + stopScreencast.mockRejectedValueOnce({ + _tag: "PreviewAutomationTimeoutError", + tabId: "recording-tab", + timeoutMs: 40, + }); + await startBrowserRecording("recording-tab"); + + await expect(stopBrowserRecording("recording-tab", 40)).rejects.toMatchObject({ + operation: "stop-deadline", + tabId: "recording-tab", + }); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set(["recording-tab"])); + + await expect(stopBrowserRecording("recording-tab")).resolves.toMatchObject({ + tabId: "recording-tab", + }); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set()); + }); + it("finishes startup before stopping so an active recording yields an artifact", async () => { let finishStartingScreencast: (() => void) | undefined; startScreencast.mockImplementationOnce(async () => { diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 2153a3ee092..06bbb03c1a0 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -95,6 +95,7 @@ interface ActiveRecording { readonly startupSettled: Promise; readonly firstFrameSize: Promise<"frame" | "cancelled">; readonly settleFirstFrameSize: (outcome: "frame" | "cancelled") => void; + artifactSave: Promise | null; recorder: MediaRecorder | null; mimeType: string | null; frameSizeEstablished: boolean; @@ -317,11 +318,14 @@ const waitForRecordingStartupToSettle = async (recording: ActiveRecording): Prom const isStartupWaitTimeout = (error: unknown): error is BrowserRecordingOperationError => isBrowserRecordingOperationError(error) && error.operation === "wait-startup"; -const recordingStopDeadlineError = (tabId: string): BrowserRecordingOperationError => +const recordingStopDeadlineError = ( + tabId: string, + cause: unknown = new Error(`Browser recording stop exceeded its deadline for tab ${tabId}.`), +): BrowserRecordingOperationError => new BrowserRecordingOperationError({ operation: "stop-deadline", tabId, - cause: new Error(`Browser recording stop exceeded its deadline for tab ${tabId}.`), + cause, }); export const isBrowserRecordingStopDeadlineError = ( @@ -329,6 +333,24 @@ export const isBrowserRecordingStopDeadlineError = ( ): error is BrowserRecordingOperationError => isBrowserRecordingOperationError(error) && error.operation === "stop-deadline"; +const isDesktopRecordingTimeout = (error: unknown): boolean => { + const seen = new Set(); + let current = error; + while (current && typeof current === "object" && !seen.has(current)) { + seen.add(current); + if ( + ("_tag" in current && + (current as { readonly _tag?: unknown })._tag === "PreviewAutomationTimeoutError") || + ("name" in current && + (current as { readonly name?: unknown }).name === "PreviewAutomationTimeoutError") + ) { + return true; + } + current = "cause" in current ? (current as { readonly cause?: unknown }).cause : undefined; + } + return false; +}; + const remainingRecordingStopBudget = ( deadline: number | null, tabId: string, @@ -422,6 +444,7 @@ export async function startBrowserRecording( startupSettled, firstFrameSize, settleFirstFrameSize: (outcome) => settleFirstFrameSize?.(outcome), + artifactSave: null, recorder: null, mimeType: null, frameSizeEstablished: false, @@ -587,6 +610,7 @@ const finalizeBrowserRecording = async ( ); } catch (cause) { if (isBrowserRecordingStopDeadlineError(cause)) throw cause; + if (isDesktopRecordingTimeout(cause)) throw recordingStopDeadlineError(tabId, cause); throw new BrowserRecordingOperationError({ operation: "stop-screencast", tabId, @@ -612,20 +636,28 @@ const finalizeBrowserRecording = async ( } try { const blob = new Blob(recording.chunks, { type: recording.mimeType }); - const data = new Uint8Array( - await awaitWithinRecordingStopDeadline(blob.arrayBuffer(), deadline, tabId), - ); - const saveBudget = remainingRecordingStopBudget(deadline, tabId); - const artifact = await awaitWithinRecordingStopDeadline( - saveBudget === undefined - ? bridge.recording.save(tabId, recording.mimeType, data) - : bridge.recording.save(tabId, recording.mimeType, data, saveBudget), - deadline, - tabId, - ); + let artifactSave = recording.artifactSave; + if (!artifactSave) { + const data = new Uint8Array( + await awaitWithinRecordingStopDeadline(blob.arrayBuffer(), deadline, tabId), + ); + const saveBudget = remainingRecordingStopBudget(deadline, tabId); + const saveOperation = + saveBudget === undefined + ? bridge.recording.save(tabId, recording.mimeType, data) + : bridge.recording.save(tabId, recording.mimeType, data, saveBudget); + const trackedSave = saveOperation.catch((cause) => { + if (recording.artifactSave === trackedSave) recording.artifactSave = null; + throw cause; + }); + recording.artifactSave = trackedSave; + artifactSave = trackedSave; + } + const artifact = await awaitWithinRecordingStopDeadline(artifactSave, deadline, tabId); result = { _tag: "Success", artifact }; } catch (cause) { if (isBrowserRecordingStopDeadlineError(cause)) throw cause; + if (isDesktopRecordingTimeout(cause)) throw recordingStopDeadlineError(tabId, cause); throw new BrowserRecordingOperationError({ operation: "save-artifact", tabId, From b58462046a2e1835e84801f68d01d5c0105160d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Miguel?= Date: Tue, 18 Aug 2026 01:52:55 +0100 Subject: [PATCH 43/43] fix(preview): make deadline retries idempotent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Bound failed recording-start cleanup and always release its slot - Retry appearance changes when a replacement rejects the stale command - Reuse validated artifact keys across desktop save timeout retries 🤖 Co-authored by GPT-5 in Codex via T3 Code --- BRANCH_DETAILS.md | 4 +- apps/desktop/src/ipc/methods/preview.ts | 3 +- apps/desktop/src/preload.ts | 3 +- apps/desktop/src/preview/Manager.test.ts | 35 +++++++-- apps/desktop/src/preview/Manager.ts | 16 +++-- apps/web/src/browser/browserRecording.test.ts | 72 ++++++++++++++++--- apps/web/src/browser/browserRecording.ts | 49 +++++++++++-- packages/contracts/src/ipc.test.ts | 19 ++++- packages/contracts/src/ipc.ts | 7 ++ 9 files changed, 178 insertions(+), 30 deletions(-) diff --git a/BRANCH_DETAILS.md b/BRANCH_DETAILS.md index c2decd3d735..4f25ef297cd 100644 --- a/BRANCH_DETAILS.md +++ b/BRANCH_DETAILS.md @@ -20,7 +20,7 @@ This branch does not add a second recording/PiP capture lifecycle or another hid Expected behavior: - Every Electron automation operation has a bounded control-session lifetime. The desktop manager reserves response grace inside the requested timeout without making the execution budget shrink when the caller increases a short timeout, always finalizes controller and action-timeline state, and detaches a timed-out debugger session while still holding an acquired control permit when a CDP command may be pending. Session removal and debugger teardown are atomic with respect to new session acquisition and bound to the exact acquired session, so late interruption or snapshot cleanup cannot detach a healthy replacement. Operations already queued on the retired semaphore detect that stale session and retry against its replacement. A request that times out while queued behind another action does not detach that action's shared debugger session. -- Click, type, press, scroll, and wait operations clamp their caller-supplied timeout to the remaining renderer host budget before entering the desktop control-session boundary. Color-scheme changes and recording startup likewise receive the remaining deadline after overlay readiness; a timed-out color-scheme command does not persist a late preference, and timed-out recording startup tears down its frame-capture session and renderer recording state. Timed appearance persistence re-reads current tab state after CDP settles and reapplies to the current guest if the webview was replaced instead of trusting a pre-await target snapshot. Recording stop bounds desktop capture shutdown, MediaRecorder settlement, blob conversion, and artifact persistence to the remaining deadline; renderer and desktop-originated deadline failures retain captured chunks and the recording slot so finalization can be retried instead of silently losing the artifact. An in-flight artifact-save promise is shared with that retry, preventing duplicate timestamped files when the renderer deadline wins before desktop IPC settles. Operations without a caller timeout use the remaining bounded request budget rather than restarting the desktop default after renderer readiness work. +- Click, type, press, scroll, and wait operations clamp their caller-supplied timeout to the remaining renderer host budget before entering the desktop control-session boundary. Color-scheme changes and recording startup likewise receive the remaining deadline after overlay readiness; a timed-out color-scheme command does not persist a late preference, and timed-out recording startup bounds cleanup to that deadline while always releasing its renderer recording slot. Timed appearance persistence re-reads current tab state after CDP settles and retries against the current guest if replacement rejects the stale guest's command. Recording stop bounds desktop capture shutdown, MediaRecorder settlement, blob conversion, and artifact persistence to the remaining deadline; renderer and desktop-originated deadline failures retain captured chunks and the recording slot so finalization can be retried instead of silently losing the artifact. An in-flight artifact-save promise is shared with that retry, and every retry reuses a validated desktop artifact idempotency key, preventing duplicate files whether the renderer deadline wins before desktop IPC settles or the desktop reports a timeout after writing. Operations without a caller timeout use the remaining bounded request budget rather than restarting the desktop default after renderer readiness work. - Snapshot collection keeps active-tab capture on CDP `Page.captureScreenshot` from the compositor surface. For an unselected tab, the renderer stages the still-mounted guest at effectively transparent opacity for two compositor frames, but only for the snapshot itself. The desktop manager captures that compositor surface without focusing the guest or calling `Page.bringToFront`; either activation call can make Electron promote the native guest over the host window and keep the T3 interface covered after staging ends. A separately bounded `webContents.capturePage` attempt provides a fallback, using `stayHidden: true` for background guests and normal visible-page capture for the foreground. Primary and fallback screenshot waits are clamped to the remaining control-session deadline, with budget reserved for fallback and result settlement, so a tight caller deadline can still return semantic data instead of being preempted by the outer session timeout. Every returned PNG, including resized output, is validated and bounded. Final screenshot failure or timeout is logged, an actually timed-out CDP capture resets the session before releasing its control permit, queued work reattaches before issuing its first command, and a capture skipped before CDP runs leaves the healthy session attached. The semantic page state, interactive elements, accessibility tree, diagnostics, and action timeline still return with `screenshot: null` instead of failing the complete snapshot. - Desktop preview guests following the system color scheme create their CDP debugger session lazily, with initialization included in the automation operation deadline. This prevents an offscreen Chromium guest from leaving `Runtime.enable` pending while holding the synchronized session lock, which previously made every later evaluation or snapshot against that tab time out even after it became presentable. `apps/web/src/browser/desktopTabLifetime.ts` passes the upstream browser appearance default through `DesktopPreviewCreateTabInputSchema` in `packages/contracts/src/ipc.ts`; `apps/desktop/src/preview/Manager.ts` normalizes that value. A non-system color-scheme override is restored after webview registration or detached DevTools closes through a separately bounded recovery path, while tabs following the system scheme stay detached until the next automation operation. - Building on upstream's retained hidden guest, automation background snapshot presentation is reference-counted independently from the normal surface lease and composes with upstream's fitted-source content and corner-radius presentation. `PreviewAutomationHosts.tsx` passes the epoch-scoped runtime tab id into `previewAutomationPresentation.ts`; every surface lookup, staging marker, readiness check, diagnostic read, lease, and desktop capture targets that exact runtime guest, while selection and errors retain the stable server tab id. The presentation helper API has no state-derived or server-id compatibility fallback. Only a one-shot automation snapshot acquires this lease; upstream recording and picture-in-picture continue to use their shared frame-capture lifecycle, while navigation, color-scheme changes, evaluation, waits, and input operations do not acquire an automation presentation lease. Staging always restores the offscreen position and does not change the human-selected surface. The entire lease, including compositor-frame staging and desktop IPC, is bounded by the operation's remaining response budget and reports a typed timeout if it stalls. If the server epoch replaces the runtime guest while staging is pending, the snapshot fails immediately with `PreviewAutomationTargetUnavailableError` instead of waiting on the stale staging marker. If the user foregrounds the target in either surface while staging is pending, that visible presentation satisfies readiness. A never-presented tab does not depend on another browser surface having supplied a panel rectangle: automation staging falls back to a deterministic rectangle fitted inside the renderer viewport. @@ -76,7 +76,7 @@ vp test run scripts/dev-runner.test.ts apps/desktop/src/app/DesktopAppIdentity.t Current verification: -- The branch-focused suite passed 284 tests across 19 files on Windows, including bounded post-overlay and queued resize mutations, retryable and deduplicated recording finalization across renderer and desktop timeouts, recording-start cleanup, replacement-guest appearance persistence, short-deadline viewport polling, post-read runtime-replacement, and background-guest accessibility regressions. The two unchanged desktop path-fixture files in the command above currently produce eight POSIX-versus-Windows path assertion failures; the affected runtime behavior passes in the remaining focused suite. +- The branch-focused suite passed 287 tests across 19 files on Windows, including bounded post-overlay and queued resize mutations, retryable and idempotent recording finalization across renderer and desktop timeouts, deadline-bounded recording-start cleanup, replacement-guest appearance retries, short-deadline viewport polling, post-read runtime-replacement, and background-guest accessibility regressions. The two unchanged desktop path-fixture files in the command above currently produce eight POSIX-versus-Windows path assertion failures; the affected runtime behavior passes in the remaining focused suite. - The incoming browser-default, provider-access, pull-request budget, tooltip, update-copy, and settings coverage passed 552 tests across 26 files. The focused open-policy and open-session subset passes 18 tests, including a shared explicit-over-default presentation decision and a reused rendered tab that remains in the background without a visibility wait. - Desktop, server, contracts, and mobile typechecks pass. Web typecheck currently reports 16 existing errors in `BranchToolbarBranchSelector.tsx`, `ModelPickerContent.tsx`, `PreviewAutomationHosts.tsx` at the pre-existing registry access, `FontFamilyPicker.tsx`, `use-atom-command.ts`, and `use-atom-query-runner.ts`; none are in the browser-default integration changed here. - An isolated web client on ports `5744`/`13784` paired and loaded successfully. Settings → Integrations exposed the browser defaults and agent-access setting; agent browser access could be disabled and restored. The non-Electron client correctly disabled desktop-only viewport, zoom, and appearance controls. diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 3d81e7a9843..f6949677b5c 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -376,10 +376,11 @@ export const saveRecording = DesktopIpc.makeIpcMethod({ tabId, mimeType, data, + idempotencyKey, timeoutMs, }) { const manager = yield* PreviewManager.PreviewManager; - return yield* manager.saveRecording(tabId, mimeType, data, timeoutMs); + return yield* manager.saveRecording(tabId, mimeType, data, idempotencyKey, timeoutMs); }), }); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 36f1cb8b99a..07aaa5f972a 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -215,11 +215,12 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_START_CHANNEL, { tabId, timeoutMs }), stopScreencast: (tabId, timeoutMs) => ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_STOP_CHANNEL, { tabId, timeoutMs }), - save: (tabId, mimeType, data, timeoutMs) => + save: (tabId, mimeType, data, idempotencyKey, timeoutMs) => ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_SAVE_CHANNEL, { tabId, mimeType, data, + idempotencyKey, timeoutMs, }), onFrame: (listener) => { diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 42d328accb7..0b5a89800d7 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1861,18 +1861,18 @@ describe("PreviewManager", () => { ), ); - effectIt.effect("reapplies a bounded color scheme after the webview is replaced", () => + effectIt.effect("retries a bounded color scheme when replacement rejects the old command", () => withManager((manager) => Effect.gen(function* () { - let finishFirstGuest: (() => void) | undefined; + let rejectFirstGuest: ((cause: unknown) => void) | undefined; const firstSendCommand = vi.fn( async (method: string, parameters?: { features?: ReadonlyArray<{ value: string }> }) => { if ( method === "Emulation.setEmulatedMedia" && parameters?.features?.[0]?.value === "dark" ) { - await new Promise((resolve) => { - finishFirstGuest = resolve; + await new Promise((_resolve, reject) => { + rejectFirstGuest = reject; }); } return undefined; @@ -1907,6 +1907,7 @@ describe("PreviewManager", () => { }), detach: vi.fn(() => { attached = false; + if (id === 42) rejectFirstGuest?.(new Error("old guest detached")); }), sendCommand, on: vi.fn(), @@ -1933,7 +1934,6 @@ describe("PreviewManager", () => { yield* Effect.yieldNow; yield* manager.registerWebview("tab_scheme_replacement", 43); yield* Effect.yieldNow; - finishFirstGuest?.(); yield* Fiber.join(mutation); expect(replacementSendCommand).toHaveBeenCalledWith("Emulation.setEmulatedMedia", { @@ -1944,6 +1944,31 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("uses an idempotent artifact path for recording save retries", () => + withManager((manager) => + Effect.gen(function* () { + const data = new Uint8Array([1, 2, 3]); + const first = yield* manager.saveRecording( + "tab_recording_save", + "video/webm", + data, + "8edc2f33-7bb4-4a30-97e8-e78f1d84513a", + ); + const retry = yield* manager.saveRecording( + "tab_recording_save", + "video/webm", + data, + "8edc2f33-7bb4-4a30-97e8-e78f1d84513a", + ); + + expect(retry.id).toBe(first.id); + expect(retry.path).toBe(first.path); + expect(writeFile).toHaveBeenCalledTimes(2); + expect(writeFile.mock.calls[1]?.[0]).toBe(writeFile.mock.calls[0]?.[0]); + }), + ), + ); + effectIt.effect("blocks late webview and capture starts during tab close", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 2fb05b39264..acead8c61d9 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2345,7 +2345,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (remainingTimeoutMs <= 0) { return yield* new PreviewAutomationTimeoutError({ tabId, timeoutMs }); } - yield* withControlSession( + const commandExit = yield* withControlSession( tabId, target, "set-color-scheme", @@ -2359,7 +2359,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ], }), remainingTimeoutMs, - ); + ).pipe(Effect.exit); const currentTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); if (!currentTab) { return yield* new PreviewTabNotFoundError({ tabId }); @@ -2368,6 +2368,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function target = yield* requireWebContents(tabId); continue; } + if (Exit.isFailure(commandExit)) { + return yield* Effect.failCause(commandExit.cause); + } if (currentTab.colorScheme !== colorScheme) { yield* update(tabId, { colorScheme }); } @@ -2956,9 +2959,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, mimeType: string, data: Uint8Array, + idempotencyKey: string, ) { - const [createdAt, millis] = yield* Effect.all([currentIso, currentMillis]); - const id = `browser-recording-${millis.toString(36)}`; + const createdAt = yield* currentIso; + const id = `browser-recording-${idempotencyKey}`; const extension = mimeType.includes("mp4") ? "mp4" : "webm"; const artifactPath = path.join(resolvedArtifactDirectory, `${id}.${extension}`); yield* fileSystem.makeDirectory(resolvedArtifactDirectory, { recursive: true }).pipe( @@ -2997,9 +3001,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, mimeType: string, data: Uint8Array, + idempotencyKey: string, timeoutMs?: number, ) { - const save = performSaveRecording(tabId, mimeType, data); + const save = performSaveRecording(tabId, mimeType, data, idempotencyKey); if (timeoutMs === undefined) return yield* save; const result = yield* save.pipe(Effect.timeoutOption(automationExecutionBudget(timeoutMs))); if (Option.isSome(result)) return result.value; @@ -4376,6 +4381,7 @@ export class PreviewManager extends Context.Service< tabId: string, mimeType: string, data: Uint8Array, + idempotencyKey: string, timeoutMs?: number, ) => Effect.Effect; readonly automationStatus: ( diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index dc44da590a1..cf7af209a2b 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -39,14 +39,22 @@ const { value.tabIds.size === 0 ? "clear" : `publish:${Array.from(value.tabIds).join(",")}`, ); }), - save: vi.fn(async (tabId: string) => ({ - id: "recording-test", - tabId, - path: "/tmp/recording-test.webm", - mimeType: "video/webm" as const, - sizeBytes: 0, - createdAt: "2026-06-26T00:00:00.000Z", - })), + save: vi.fn( + async ( + tabId: string, + _mimeType?: string, + _data?: Uint8Array, + _idempotencyKey?: string, + _timeoutMs?: number, + ) => ({ + id: "recording-test", + tabId, + path: "/tmp/recording-test.webm", + mimeType: "video/webm" as const, + sizeBytes: 0, + createdAt: "2026-06-26T00:00:00.000Z", + }), + ), startScreencast: vi.fn(async (tabId: string) => { events.push("start-screencast"); const surface = surfaceState.byTabId[tabId] as @@ -237,7 +245,30 @@ describe("browser recording", () => { await rejection; expect(startScreencast).toHaveBeenCalledWith("recording-tab", 40); - expect(stopScreencast).toHaveBeenCalledWith("recording-tab"); + expect(stopScreencast).toHaveBeenCalledWith("recording-tab", 1); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set()); + }); + + it("clears a timed-out startup even when screencast cleanup stalls", async () => { + vi.useFakeTimers(); + startScreencast.mockImplementationOnce(async () => { + events.push("start-screencast"); + }); + stopScreencast.mockImplementationOnce( + async () => await new Promise(() => undefined), + ); + + const startPromise = startBrowserRecording("recording-tab", null, "recording-tab", 40); + const rejection = expect(startPromise).rejects.toMatchObject({ + operation: "wait-first-frame", + tabId: "recording-tab", + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(40); + await vi.runOnlyPendingTimersAsync(); + + await rejection; + expect(stopScreencast).toHaveBeenCalledWith("recording-tab", 1); expect(readActiveBrowserRecordingTabIds()).toEqual(new Set()); }); @@ -578,6 +609,29 @@ describe("browser recording", () => { expect(readActiveBrowserRecordingTabIds()).toEqual(new Set()); }); + it("reuses the artifact idempotency key after a desktop save timeout", async () => { + save.mockRejectedValueOnce({ + _tag: "PreviewAutomationTimeoutError", + tabId: "recording-tab", + timeoutMs: 40, + }); + await startBrowserRecording("recording-tab"); + + await expect(stopBrowserRecording("recording-tab", 40)).rejects.toMatchObject({ + operation: "stop-deadline", + tabId: "recording-tab", + }); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set(["recording-tab"])); + + await expect(stopBrowserRecording("recording-tab")).resolves.toMatchObject({ + tabId: "recording-tab", + }); + expect(save).toHaveBeenCalledTimes(2); + expect(save.mock.calls[0]?.[3]).toEqual(expect.any(String)); + expect(save.mock.calls[1]?.[3]).toBe(save.mock.calls[0]?.[3]); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set()); + }); + it("finishes startup before stopping so an active recording yields an artifact", async () => { let finishStartingScreencast: (() => void) | undefined; startScreencast.mockImplementationOnce(async () => { diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 06bbb03c1a0..7b6a1531688 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -8,6 +8,7 @@ import * as Schema from "effect/Schema"; import { Atom } from "effect/unstable/reactivity"; import { previewBridge } from "~/components/preview/previewBridge"; +import { randomUUID } from "~/lib/utils"; import { appAtomRegistry } from "~/rpc/atomRegistry"; import { useBrowserSurfaceStore } from "./browserSurfaceStore"; @@ -95,6 +96,7 @@ interface ActiveRecording { readonly startupSettled: Promise; readonly firstFrameSize: Promise<"frame" | "cancelled">; readonly settleFirstFrameSize: (outcome: "frame" | "cancelled") => void; + readonly artifactSaveKey: string; artifactSave: Promise | null; recorder: MediaRecorder | null; mimeType: string | null; @@ -238,10 +240,38 @@ const clearActiveRecording = (recording: ActiveRecording): void => { const cleanupFailedRecordingStart = async ( bridge: NonNullable, recording: ActiveRecording, + deadline: number | null, ): Promise => { const errors: unknown[] = []; try { - await bridge.recording.stopScreencast(recording.tabId); + const remainingMs = deadline === null ? undefined : Math.max(0, deadline - Date.now()); + const stop = + remainingMs === undefined + ? bridge.recording.stopScreencast(recording.tabId) + : bridge.recording.stopScreencast(recording.tabId, Math.max(1, remainingMs)); + if (remainingMs === undefined) { + await stop; + } else { + let timeout: ReturnType | null = null; + try { + await Promise.race([ + stop, + new Promise((_, reject) => { + timeout = setTimeout( + () => + reject( + new Error( + `Browser recording startup cleanup exceeded its deadline for tab ${recording.tabId}.`, + ), + ), + remainingMs, + ); + }), + ]); + } finally { + if (timeout !== null) clearTimeout(timeout); + } + } } catch (error) { errors.push(error); } @@ -444,6 +474,7 @@ export async function startBrowserRecording( startupSettled, firstFrameSize, settleFirstFrameSize: (outcome) => settleFirstFrameSize?.(outcome), + artifactSaveKey: randomUUID(), artifactSave: null, recorder: null, mimeType: null, @@ -509,7 +540,7 @@ export async function startBrowserRecording( ? `Browser recording startup exceeded its deadline for tab ${tabId}.` : `No valid recording frame arrived for tab ${tabId}.`, ); - const cleanupCause = await cleanupFailedRecordingStart(bridge, recording); + const cleanupCause = await cleanupFailedRecordingStart(bridge, recording, deadline); throw new BrowserRecordingOperationError({ operation: "wait-first-frame", tabId, @@ -538,7 +569,7 @@ export async function startBrowserRecording( if (event.data.size > 0) chunks.push(event.data); }); } catch (cause) { - const cleanupCause = await cleanupFailedRecordingStart(bridge, recording); + const cleanupCause = await cleanupFailedRecordingStart(bridge, recording, deadline); throw new BrowserRecordingOperationError({ operation: "initialize-media-recorder", tabId, @@ -555,7 +586,7 @@ export async function startBrowserRecording( try { recorder.start(1_000); } catch (cause) { - const cleanupCause = await cleanupFailedRecordingStart(bridge, recording); + const cleanupCause = await cleanupFailedRecordingStart(bridge, recording, deadline); throw new BrowserRecordingOperationError({ operation: "start-media-recorder", tabId, @@ -644,8 +675,14 @@ const finalizeBrowserRecording = async ( const saveBudget = remainingRecordingStopBudget(deadline, tabId); const saveOperation = saveBudget === undefined - ? bridge.recording.save(tabId, recording.mimeType, data) - : bridge.recording.save(tabId, recording.mimeType, data, saveBudget); + ? bridge.recording.save(tabId, recording.mimeType, data, recording.artifactSaveKey) + : bridge.recording.save( + tabId, + recording.mimeType, + data, + recording.artifactSaveKey, + saveBudget, + ); const trackedSave = saveOperation.catch((cause) => { if (recording.artifactSave === trackedSave) recording.artifactSave = null; throw cause; diff --git a/packages/contracts/src/ipc.test.ts b/packages/contracts/src/ipc.test.ts index b0200bdcd55..8765d7b7e48 100644 --- a/packages/contracts/src/ipc.test.ts +++ b/packages/contracts/src/ipc.test.ts @@ -52,12 +52,29 @@ describe("desktop recording finalization deadlines", () => { tabId: "tab-1", timeoutMs: 1_250, }); - expect(decodeSave({ tabId: "tab-1", mimeType: "video/webm", data, timeoutMs: 1_250 })).toEqual({ + expect( + decodeSave({ + tabId: "tab-1", + mimeType: "video/webm", + data, + idempotencyKey: "f3088f18-9595-44f8-a67c-50c587d034a2", + timeoutMs: 1_250, + }), + ).toEqual({ tabId: "tab-1", mimeType: "video/webm", data, + idempotencyKey: "f3088f18-9595-44f8-a67c-50c587d034a2", timeoutMs: 1_250, }); + expect(() => + decodeSave({ + tabId: "tab-1", + mimeType: "video/webm", + data, + idempotencyKey: "../unsafe", + }), + ).toThrow(); }); }); diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 0da2c72c744..d0cda1cdf7f 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -1026,6 +1026,12 @@ export const DesktopPreviewRecordingSaveInputSchema = Schema.Struct({ tabId: DesktopPreviewTabIdSchema, mimeType: Schema.String.check(Schema.isTrimmed()).check(Schema.isNonEmpty()), data: Schema.Uint8Array, + idempotencyKey: Schema.String.check( + Schema.isTrimmed(), + Schema.isNonEmpty(), + Schema.isMaxLength(128), + Schema.isPattern(/^[a-z0-9-]+$/i), + ), timeoutMs: Schema.optional( Schema.Int.check(Schema.isGreaterThan(0)).check(Schema.isLessThanOrEqualTo(60_000)), ), @@ -1209,6 +1215,7 @@ export interface DesktopPreviewBridge { tabId: string, mimeType: string, data: Uint8Array, + idempotencyKey: string, timeoutMs?: number, ) => Promise; onFrame: (listener: (frame: DesktopPreviewRecordingFrame) => void) => () => void;