feat(ios): route simulator snapshots through AX bridge

Author: thymikee

README.md

@@ -141,7 +141,7 @@ The same session and evidence model works at every step: the agent explores the
 
 ## How it works
 
-`agent-device` keeps device state in sessions. It sends commands to XCTest on iOS and tvOS, ADB and the snapshot helper on Android, HDC and ArkUI `uitest` on HarmonyOS, Vega CLI/VDA on the Vega Virtual Device, a local helper on macOS, and AT-SPI on Linux.
+`agent-device` keeps device state in sessions. It uses a local accessibility bridge for iOS Simulator snapshots and XCTest for iOS interactions, physical iOS, and tvOS; ADB and the snapshot helper on Android; HDC and ArkUI `uitest` on HarmonyOS; Vega CLI/VDA on the Vega Virtual Device; a local helper on macOS; and AT-SPI on Linux.
 
 Support depth varies by target. Newer backends such as HarmonyOS and Vega OS cover a subset of commands; run `agent-device capabilities --platform <platform>` to see what a target supports.
 

docs/adr/0004-ios-snapshot-backend-strategy.md

@@ -2,17 +2,18 @@
 
 ## Status
 
-Accepted. Amended after iOS snapshot capture was simplified to two public modes:
-regular interactive snapshots and raw diagnostic snapshots.
+Accepted. Amended after local iOS Simulator acquisition moved to the host AX bridge while the
+public surface remained two modes: regular interactive snapshots and raw diagnostic snapshots.
 
-The runner owns capture-plan acquisition and backend fallback. Host-side iOS validation, semantic
-presentation, and publication are owned by `@agent-device/capture-kit`; structured snapshot quality
-verdicts make degraded or recovered output observable end to end.
+The Apple platform runtime owns acquisition routing and its generation-scoped XCTest fallback.
+Host-side iOS validation, semantic presentation, and publication are owned by
+`@agent-device/capture-kit`; structured snapshot quality verdicts and fallback warnings make
+degraded or recovered output observable end to end.
 
 ## Context
 
-Agent Device exposes iOS UI state through snapshots produced by the long-lived XCTest runner. The
-runner has two durable snapshot needs:
+Agent Device exposes iOS UI state through host AX acquisition on local Simulators and the long-lived
+XCTest runner everywhere else. The snapshot surface has two durable needs:
 
 - agent-facing regular context, where the important contract is the effective user-visible UI,
   fixed controls such as tab bars, and scroll-hidden hints for content outside visible scroll
@@ -35,8 +36,13 @@ predictable.
 
 ## Decision
 
-Keep XCTest as the default iOS automation runner and split iOS snapshot capture into explicit
-strategies:
+Keep XCTest as the iOS automation runner. Route eligible local iOS Simulator snapshots through the
+host AX bridge, present them once through the shared TypeScript engine, and use one typed XCTest
+fallback when bridge acquisition or presentation fails. Disable the bridge for that app generation
+after fallback; a new app generation re-enables it. Physical devices, providers, custom-action
+captures, and interactions remain on their existing owners.
+
+Keep the two public snapshot strategies explicit:
 
 - **Regular visible strategy**: use recursive XCTest snapshots, emit the effective user-visible
   tree plus visible ancestors and scroll-hidden hints, and fall back through the capture plan when
@@ -51,12 +57,10 @@ strategies:
   carry the response, fail explicitly instead of silently truncating the tree at a hard node count.
   If XCTest reports a real AX serialization failure, preserve that error instead of pretending the
   UI is empty.
-- **Future AX-service strategy**: treat Bluesky-class failures as evidence that XCTest is
-  not a complete semantic snapshot backend. A robust semantic fix should add a host-side simulator
-  accessibility backend, similar in role to existing simulator accessibility inspection tools,
-  and acquire its output as `RawAXNode` values. Every backend crosses the same
-  `SnapshotPresentation` construction boundary before producing wire-facing `PresentedNode` values.
-  That backend can be simulator-only; physical devices should use an equivalent non-XCTest semantic
+- **Host AX strategy**: acquire local Simulator trees as raw facts through the bounded host bridge.
+  Every result crosses the same presentation boundary before publication. XCTest fallback carries
+  explicit source residue, and comparisons require matching producer, intent, app generation,
+  presentation key, and residue. Physical devices should use an equivalent non-XCTest semantic
   backend only if Apple exposes a supported channel.
 
 The daemon should make degraded output observable. If an iOS interactive snapshot contains only the

packages/capture-kit/src/ios-snapshot-planning.ts

@@ -90,6 +90,16 @@ export function areIosSnapshotComparisonIdentitiesEqual(
   );
 }
 
+export function iosSnapshotComparisonIdentityKey(identity: IosSnapshotComparisonIdentity): string {
+  return JSON.stringify({
+    producer: identity.producer,
+    intent: identity.intent,
+    lineage: identity.lineage,
+    presentationKey: identity.presentationKey,
+    residue: identity.residue.map(residueIdentity).sort(),
+  });
+}
+
 export function buildIosSnapshotComparisonIdentity(
   input: IosSnapshotInput,
   request: IosSnapshotRequest,

packages/contracts/src/interactor-types.ts

@@ -9,8 +9,9 @@ import type { SessionSurface } from './session-surface.ts';
 import type { BackendSnapshotResult } from './snapshot-types.ts';
 import type { RunnerLogicalLeaseContext } from './runner-lease-context.ts';
 import type {
-  IosProviderAcquisitionProducer,
+  IosAcquisitionProducer,
   IosSnapshotAcquisitionFacts,
+  IosSnapshotComparisonIdentity,
 } from './ios-snapshot.ts';
 import type {
   RawSnapshotNode,
@@ -182,6 +183,8 @@ export type SnapshotOptions = BaseSnapshotOptions & {
   includeRects?: boolean;
   includeHiddenContentHints?: boolean;
   surface?: SessionSurface;
+  /** Internal capture purpose; action outcomes always require the full tree. */
+  acquisitionIntent?: 'full' | 'surface-observation';
 };
 
 /**
@@ -251,11 +254,12 @@ export type KeyboardEnterResult =
  */
 export type SnapshotResult = Omit<BackendSnapshotResult, 'backend' | 'nodes'> & {
   nodes?: RawSnapshotNode[];
+  comparisonIdentity?: IosSnapshotComparisonIdentity;
 } & SnapshotProvenance;
 
 export type SnapshotRuntimeAcquiredResult = Readonly<{
   stage: 'acquired';
-  acquisition: IosSnapshotAcquisitionFacts & Readonly<{ producer: IosProviderAcquisitionProducer }>;
+  acquisition: IosSnapshotAcquisitionFacts & Readonly<{ producer: IosAcquisitionProducer }>;
 }>;
 
 export type SnapshotRuntimeResult = SnapshotResult | SnapshotRuntimeAcquiredResult;

packages/kernel/src/snapshot.ts

@@ -162,7 +162,10 @@ export type SnapshotNode = RawSnapshotNode & {
  * snapshot-provenance.test.ts).
  */
 export type SnapshotProvenance =
-  | { backend: 'xctest'; producer: 'apple-runner' | 'appium-source' | 'limrun-ios-tree' }
+  | {
+      backend: 'xctest';
+      producer: 'apple-runner' | 'simulator-ax-bridge' | 'appium-source' | 'limrun-ios-tree';
+    }
   | { backend: 'android'; producer: 'android-uiautomator' | 'appium-source' }
   | { backend: 'harmonyos-arkui'; producer: 'harmonyos-uitest' }
   | { backend: 'macos-helper'; producer: 'macos-helper' }
@@ -238,6 +241,8 @@ export type SnapshotState = {
   snapshotQuality?: SnapshotQualityVerdict;
   comparisonSafe?: boolean;
   presentationKey?: string;
+  /** Opaque equality key for iOS acquisition and presentation lineage. */
+  comparisonKey?: string;
   /**
    * Android: the capture is an occluding system surface (notification shade, quick settings)
    * rather than app content. Consumers that surface this tree to the agent must disclose the

packages/platform-apple/src/runtime-snapshot.ts

@@ -14,11 +14,13 @@ import type {
   PlatformRuntimeOperations,
 } from '@agent-device/contracts/platform-runtime-operations';
 import { isMacOs, type DeviceInfo } from '@agent-device/kernel/device';
+import type { AppleSnapshotRoute } from './snapshot-route.ts';
 
 /** Apple-owned selection between app snapshots and explicit macOS surface snapshots. */
 export function bindAppleSnapshotRuntime(
   host: PlatformRuntimeHost,
   request: Readonly<{ device: DeviceInfo; signal: AbortSignal }>,
+  route?: AppleSnapshotRoute,
 ): SnapshotRuntimeOperation {
   const appSnapshot = bindLocalSnapshotInteractor({
     device: request.device,
@@ -37,7 +39,14 @@ export function bindAppleSnapshotRuntime(
         captureSnapshotSignal(request.signal, input),
       );
     }
-    return await appSnapshot.captureSnapshot(input);
+    if (!route) return await appSnapshot.captureSnapshot(input);
+    const signal = captureSnapshotSignal(request.signal, input);
+    return await route.capture(
+      request.device,
+      input,
+      signal,
+      async (fallbackInput) => await appSnapshot.captureSnapshot(fallbackInput),
+    );
   };
   return Object.freeze({
     captureSnapshot,

packages/platform-apple/src/runtime.ts

@@ -70,6 +70,7 @@ import {
   bindAppleFindTextRuntime,
   bindAppleSnapshotRuntime,
 } from './runtime-snapshot.ts';
+import { createAppleSnapshotRoute } from './snapshot-route.ts';
 
 const owner = localRuntimeOwner('apple');
 const available = Object.freeze({ available: true } as const);
@@ -268,6 +269,7 @@ function appleFocusFact(device: DeviceInfo): RuntimeOperationFact {
 
 export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformRuntimeOwner {
   const appLogs = createAppleAppLogRuntime(host);
+  const snapshotRoute = createAppleSnapshotRoute(host);
   const inspectFacts = async (device: DeviceInfo) => {
     const logs = await appLogs.inspectFacts(device);
     const deployment = appleAppDeploymentFacts(device);
@@ -375,10 +377,14 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR
           }),
         ),
         ...whenAdmitted(facts.operations.captureSnapshot, () =>
-          bindAppleSnapshotRuntime(host, {
-            device: request.device,
-            signal: request.scope.signal,
-          }),
+          bindAppleSnapshotRuntime(
+            host,
+            {
+              device: request.device,
+              signal: request.scope.signal,
+            },
+            snapshotRoute,
+          ),
         ),
         ...whenAdmitted(facts.operations.captureScreenshot, () =>
           bindLocalScreenshotInteractor({
@@ -490,7 +496,9 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR
         [Symbol.asyncDispose]: async () => await logs[Symbol.asyncDispose](),
       }) satisfies DeviceBinding<PlatformRuntimeOperations>;
     },
-    shutdown: async () => await appLogs.shutdown(),
+    shutdown: async () => {
+      await Promise.all([appLogs.shutdown(), snapshotRoute.shutdown()]);
+    },
   });
 }