fix(preview): keep snapshots slim unless the agent asks

apps/desktop/src/ipc/methods/preview.ts

@@ -2,6 +2,7 @@ import {
   DesktopPreviewAnnotationThemeInputSchema,
   DesktopPreviewArtifactInputSchema,
   DesktopPreviewAutomationClickInputSchema,
+  DesktopPreviewAutomationSnapshotInputSchema,
   DesktopPreviewAutomationEvaluateInputSchema,
   DesktopPreviewAutomationPressInputSchema,
   DesktopPreviewAutomationScrollInputSchema,
@@ -281,11 +282,11 @@ export const automationStatus = DesktopIpc.makeIpcMethod({
 
 export const automationSnapshot = DesktopIpc.makeIpcMethod({
   channel: IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL,
-  payload: DesktopPreviewTabInputSchema,
+  payload: DesktopPreviewAutomationSnapshotInputSchema,
   result: PreviewAutomationSnapshot,
-  handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId }) {
+  handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId, include }) {
     const manager = yield* PreviewManager.PreviewManager;
-    return yield* manager.automationSnapshot(tabId);
+    return yield* manager.automationSnapshot(tabId, include ?? []);
   }),
 });
 

apps/desktop/src/preload.ts

@@ -230,8 +230,11 @@ contextBridge.exposeInMainWorld("desktopBridge", {
     automation: {
       status: (tabId) =>
         ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_STATUS_CHANNEL, { tabId }),
-      snapshot: (tabId) =>
-        ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, { tabId }),
+      snapshot: (tabId, include) =>
+        ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, {
+          tabId,
+          ...(include === undefined ? {} : { include }),
+        }),
       click: (tabId, input) =>
         ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_CLICK_CHANNEL, { tabId, input }),
       type: (tabId, input) =>

apps/desktop/src/preview/Manager.test.ts

@@ -87,6 +87,14 @@ vi.mock("electron", () => ({
   },
   nativeImage: {
     createFromPath,
+    createFromBuffer: (buffer: Buffer) => ({
+      getSize: () => ({ width: buffer.length > 0 ? 1 : 0, height: buffer.length > 0 ? 1 : 0 }),
+      toPNG: () => buffer,
+      resize: () => ({
+        getSize: () => ({ width: 1, height: 1 }),
+        toPNG: () => buffer,
+      }),
+    }),
   },
   shell: {
     showItemInFolder,
@@ -168,6 +176,7 @@ const makeTestPreviewWebContents = (
     getURL: () => "https://example.com",
     getTitle: () => "Example",
     isLoading: () => false,
+    isDevToolsOpened: () => false,
     getZoomFactor: () => 1,
     setZoomFactor: vi.fn(),
     on: vi.fn(),
@@ -2840,3 +2849,137 @@ describe("Preview automation diagnostics", () => {
     expect("locator" in error).toBe(false);
   });
 });
+
+describe("Preview automation snapshots", () => {
+  const pageValue = {
+    url: "https://example.com",
+    title: "Example",
+    loading: false,
+    visibleText: "Dashboard",
+    interactiveElements: [],
+  };
+
+  const snapshotImage = {
+    getSize: () => ({ width: 100, height: 80 }),
+    toPNG: () => Buffer.from("png"),
+    resize: () => ({
+      getSize: () => ({ width: 100, height: 80 }),
+      toPNG: () => Buffer.from("png"),
+    }),
+  };
+
+  const mockAutomationWebContents = (
+    sendCommand: ReturnType<typeof vi.fn>,
+    capturePage: () => Promise<typeof snapshotImage>,
+  ) =>
+    ({
+      id: 42,
+      isDestroyed: () => false,
+      getType: () => "webview",
+      getURL: () => "https://example.com",
+      getTitle: () => "Example",
+      isLoading: () => false,
+      isDevToolsOpened: () => false,
+      getZoomFactor: () => 1,
+      setZoomFactor: vi.fn(),
+      on: vi.fn(),
+      off: vi.fn(),
+      ipc: { on: vi.fn(), off: vi.fn() },
+      send: webviewSend,
+      navigationHistory: { canGoBack: () => false, canGoForward: () => false },
+      setWindowOpenHandler: vi.fn(),
+      capturePage,
+      debugger: {
+        isAttached: () => false,
+        attach: vi.fn(),
+        sendCommand,
+        on: vi.fn(),
+        off: vi.fn(),
+      },
+    }) as never;
+
+  effectIt.effect("omits ax, console, and network unless include asks", () =>
+    withManager((manager) =>
+      Effect.gen(function* () {
+        const sendCommand = vi.fn(async (method: string) => {
+          if (method === "Runtime.evaluate") {
+            return { result: { value: pageValue } };
+          }
+          if (method === "Accessibility.getFullAXTree") {
+            return { nodes: [{ role: "main" }] };
+          }
+          return undefined;
+        });
+        fromId.mockReturnValue(mockAutomationWebContents(sendCommand, async () => snapshotImage));
+
+        yield* manager.createTab("tab_snapshot");
+        yield* manager.registerWebview("tab_snapshot", 42);
+        const slim = yield* manager.automationSnapshot("tab_snapshot");
+        expect(slim.accessibilityTree).toBeUndefined();
+        expect(slim.consoleEntries).toEqual([]);
+        expect(slim.networkEntries).toEqual([]);
+        const slimMethods = sendCommand.mock.calls.map(([method]) => method);
+        expect(slimMethods).not.toContain("Accessibility.getFullAXTree");
+
+        sendCommand.mockClear();
+        const withAx = yield* manager.automationSnapshot("tab_snapshot", ["ax"]);
+        expect(withAx.accessibilityTree).toEqual({ nodes: [{ role: "main" }] });
+        expect(sendCommand.mock.calls.map(([method]) => method)).toContain(
+          "Accessibility.getFullAXTree",
+        );
+      }),
+    ),
+  );
+
+  effectIt.effect("fails the snapshot when capturePage and CDP screenshot both miss", () =>
+    withManager((manager) =>
+      Effect.gen(function* () {
+        const sendCommand = vi.fn(async (method: string) => {
+          if (method === "Runtime.evaluate") {
+            return { result: { value: pageValue } };
+          }
+          if (method === "Page.captureScreenshot") {
+            return {};
+          }
+          return undefined;
+        });
+        fromId.mockReturnValue(
+          mockAutomationWebContents(sendCommand, async () => {
+            throw new Error("capturePage failed");
+          }),
+        );
+
+        yield* manager.createTab("tab_snapshot_fail");
+        yield* manager.registerWebview("tab_snapshot_fail", 42);
+        const exit = yield* Effect.exit(manager.automationSnapshot("tab_snapshot_fail"));
+        expect(Exit.isFailure(exit)).toBe(true);
+        if (Exit.isSuccess(exit)) return;
+        expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({
+          _tag: "PreviewOperationError",
+          operation: "automationSnapshot.capturePage",
+        });
+      }),
+    ),
+  );
+
+  effectIt.effect("defaults waitFor text search to the main landmark", () =>
+    withManager((manager) =>
+      Effect.gen(function* () {
+        const expressions: string[] = [];
+        const sendCommand = vi.fn(async (method: string, params?: Record<string, unknown>) => {
+          if (method === "Runtime.evaluate") {
+            expressions.push(String(params?.expression ?? ""));
+            return { result: { value: { matched: true } } };
+          }
+          return undefined;
+        });
+        fromId.mockReturnValue(mockAutomationWebContents(sendCommand, async () => snapshotImage));
+
+        yield* manager.createTab("tab_wait");
+        yield* manager.registerWebview("tab_wait", 42);
+        yield* manager.automationWaitFor("tab_wait", { text: "Dashboard" });
+        expect(expressions.some((expression) => expression.includes('"main"'))).toBe(true);
+      }),
+    ),
+  );
+});