refactor(ios): route Appium snapshots through engine (#2224)

* refactor(ios): route Appium snapshots through engine

* perf(ios): keep Appium snapshot adapter lazy

* fix(ios): preserve legacy snapshot presentation boundary

* perf(ios): tighten Appium snapshot facts

* test(ios): cover legacy snapshot presentation boundary

* test(ios): cover WebDriver snapshot seams

* fix(ios): centralize WebDriver snapshot evidence

* fix(ios): preserve unavailable snapshot facts

* fix(ios): preserve snapshot error context

* fix(ios): centralize snapshot presentation ownership

* fix(ios): disclose Appium snapshot limits accurately

* fix(ios): disclose Appium snapshot evidence limits

* fix(ios): harden Appium evidence disclosure

* fix(ios): tighten Appium snapshot disclosures

* fix(ios): close Appium audit gaps

Author: thymikee

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+- Breaking (0.21): iOS Appium/WebDriver snapshots now expose engine-owned acquisition facts and
+  typed fidelity warnings. The SDK snapshot `truncated` field is optional when Appium cannot report
+  hierarchy completeness; regular snapshots fail closed without valid viewport evidence, while
+  `snapshot --raw` remains available for diagnostics (#2195).
 - Security (daemon, remote/proxy HTTP only): when `AGENT_DEVICE_HTTP_AUTH_HOOK` is configured and a
   request's hook result does not attest a `tenantId`, the request is now refused (401) outright — the
   daemon no longer runs it as whichever tenant the client declared (RPC body `meta.tenantId` or

packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts

@@ -7,7 +7,7 @@ import {
   createIosSnapshotRequest,
   deriveIosCaptureHint,
 } from '@agent-device/capture-kit/ios-snapshot-planning';
-import { IosSnapshotEngineError, presentIosSnapshot } from './index.ts';
+import { IosSnapshotEngineError, presentIosSnapshot, publishIosSnapshot } from './index.ts';
 import { runTypeScriptCase, writeDifferentialFailureArtifact } from './conformance-harness.ts';
 import {
   acquisitionForGoldenCase,
@@ -86,7 +86,7 @@ test('the independent iOS snapshot goldens match the TypeScript engine', () => {
       | {
           outcome: 'success';
           nodes: ReturnType<typeof normalizeGoldenNodes>;
-          truncated: boolean;
+          truncated?: boolean;
           residue: typeof acquisition.residue;
           qualityLabels?: readonly (string | null)[];
         }
@@ -97,16 +97,21 @@ test('the independent iOS snapshot goldens match the TypeScript engine', () => {
         };
 
     try {
-      const result = presentIosSnapshot({ stage: 'acquired', acquisition }, request, {
+      const presentation = presentIosSnapshot({ stage: 'acquired', acquisition }, request, {
+        foldPolicy: testCase.foldPolicy,
+      });
+      const publication = publishIosSnapshot({ stage: 'acquired', acquisition }, request, {
         foldPolicy: testCase.foldPolicy,
       });
       actual = {
         outcome: 'success',
-        nodes: normalizeGoldenNodes(result.nodes),
-        truncated: acquisition.truncated,
-        residue: acquisition.residue,
+        nodes: normalizeGoldenNodes(publication.payload.nodes),
+        ...(publication.payload.truncated === undefined
+          ? {}
+          : { truncated: publication.payload.truncated }),
+        residue: publication.residue,
         ...(testCase.qualityLabels
-          ? { qualityLabels: result.qualityNodes?.map((node) => node.label ?? null) }
+          ? { qualityLabels: presentation.qualityNodes?.map((node) => node.label ?? null) }
           : {}),
       };
     } catch (error) {
@@ -125,6 +130,21 @@ test('the independent iOS snapshot goldens match the TypeScript engine', () => {
   }
 });
 
+test('published payload omits unknown truncation instead of defaulting it', () => {
+  const fixture = readIosSnapshotEngineFixture();
+  const testCase = fixture.cases[0]!;
+  const request = requestForGoldenCase(testCase);
+  const acquisition = {
+    ...acquisitionForGoldenCase(fixture, testCase),
+    truncated: undefined,
+  };
+
+  const publication = publishIosSnapshot({ stage: 'acquired', acquisition }, request);
+
+  assert.equal(publication.payload.truncated, undefined);
+  assert.equal('truncated' in publication.payload, false);
+});
+
 test('the differential TypeScript runner preserves typed failures', () => {
   const fixture = readIosSnapshotEngineFixture();
   const source = fixture.cases.find(

packages/capture-kit/src/ios-snapshot-engine/engine.test.ts

@@ -11,12 +11,14 @@ import {
   createIosSnapshotRequest,
   deriveIosCaptureHint,
 } from '@agent-device/capture-kit/ios-snapshot-planning';
+import { toIosSnapshotEngineErrorDetails } from './types.ts';
 import {
   compactIosInteractiveSnapshot,
   createIosSnapshotEngine,
   IosSnapshotEngineError,
   presentIosSnapshot,
   publishIosSnapshot,
+  resolveIosViewportEvidenceFromRoots,
 } from './index.ts';
 import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot';
 
@@ -189,6 +191,65 @@ test('regular presentation fails typed when the viewport is missing or the graph
   );
 });
 
+test('engine error details preserve typed public context', () => {
+  const error = new IosSnapshotEngineError('invalid-viewport', 'invalid viewport', {
+    field: 'viewport',
+    index: 4,
+    parentIndex: 2,
+    frame: { x: 0, y: 0, width: 10, height: 10 },
+    clip: { x: 1, y: 2, width: 3, height: 4 },
+    projection: 'regular',
+  });
+
+  assert.deepEqual(toIosSnapshotEngineErrorDetails(error), {
+    reason: 'invalid-viewport',
+    field: 'viewport',
+    index: 4,
+    parentIndex: 2,
+    frame: { x: 0, y: 0, width: 10, height: 10 },
+    clip: { x: 1, y: 2, width: 3, height: 4 },
+    projection: 'regular',
+  });
+});
+
+test('viewport evidence prefers reported application and window roots', () => {
+  assert.deepEqual(
+    resolveIosViewportEvidenceFromRoots([
+      { type: 'XCUIElementTypeApplication', rectStatus: 'invalid' },
+      {
+        type: 'XCUIElementTypeWindow',
+        rect: { x: 0, y: 0, width: 390, height: 844 },
+        rectStatus: 'reported',
+      },
+    ]),
+    { kind: 'reported', rect: { x: 0, y: 0, width: 390, height: 844 } },
+  );
+});
+
+test('viewport evidence can fall back to the largest top-level root', () => {
+  assert.deepEqual(
+    resolveIosViewportEvidenceFromRoots(
+      [
+        { type: 'Other', rect: { x: 0, y: 0, width: 100, height: 100 } },
+        { type: 'Other', rect: { x: 0, y: 0, width: 200, height: 300 } },
+      ],
+      { fallbackToLargestRoot: true },
+    ),
+    { kind: 'reported', rect: { x: 0, y: 0, width: 200, height: 300 } },
+  );
+});
+
+test('viewport evidence preserves explicit missing geometry reasons', () => {
+  assert.deepEqual(
+    resolveIosViewportEvidenceFromRoots([{ type: 'Application', rectStatus: 'invalid' }]),
+    { kind: 'missing', reason: 'invalid' },
+  );
+  assert.deepEqual(
+    resolveIosViewportEvidenceFromRoots([{ type: 'Application', rectStatus: 'not-provided' }]),
+    { kind: 'missing', reason: 'not-provided' },
+  );
+});
+
 test('presented runner payloads and optional quality payloads cross the host invariant', () => {
   const request = createIosSnapshotRequest();
   const presentedCapture = publishIosSnapshot(acquiredInput(request, nestedNodes()), request);
@@ -234,10 +295,12 @@ test('unavailable hittability never becomes regular actionability', () => {
     residue: [{ kind: 'unavailable-fact' as const, fact: 'hittability' as const }],
   } satisfies IosSnapshotAcquisition;
   const acquired = publishIosSnapshot({ stage: 'acquired', acquisition: unavailable }, request);
-  assert.equal(
-    acquired.payload.nodes.find((node) => node.label === 'Partially visible')?.hittable,
-    false,
+  const partiallyVisible = acquired.payload.nodes.find(
+    (node) => node.label === 'Partially visible',
   );
+  assert.ok(partiallyVisible);
+  assert.equal(partiallyVisible.hittable, undefined);
+  assert.equal('hittable' in partiallyVisible, false);
 
   const available = publishIosSnapshot(acquiredInput(request, nestedNodes()), request);
   const presented: IosSnapshotInput = {

packages/capture-kit/src/ios-snapshot-engine/engine.ts

@@ -11,8 +11,11 @@ import type {
   IosSnapshotInput,
   IosSnapshotPublication,
   IosSnapshotRequest,
+  IosViewportEvidence,
 } from '@agent-device/contracts/ios-snapshot';
-import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot';
+import { normalizeType } from '@agent-device/contracts/snapshot';
+import { isPositiveFiniteRect } from '@agent-device/kernel/rect';
+import { attachRefs, type RawSnapshotNode, type Rect } from '@agent-device/kernel/snapshot';
 import { buildIosInteractiveSnapshotPresentation } from './semantic-index.ts';
 import { validateIosSnapshotGraph } from './graph.ts';
 import { foldIosSnapshot } from './geometry.ts';
@@ -28,6 +31,52 @@ import { IosSnapshotEngineError } from './types.ts';
 
 const DEFAULT_FOLD_POLICY: IosSnapshotFoldPolicy = 'cursor-projected';
 
+export type IosSnapshotViewportRoot = Readonly<{
+  type?: string;
+  rect?: Rect;
+  rectStatus?: 'reported' | 'invalid' | 'not-provided';
+}>;
+
+export function resolveIosViewportEvidenceFromRoots(
+  roots: readonly IosSnapshotViewportRoot[],
+  options: Readonly<{ fallbackToLargestRoot?: boolean }> = {},
+): IosViewportEvidence | undefined {
+  const viewportRoots = roots.filter(isViewportRoot);
+  const candidates =
+    viewportRoots.length > 0 || options.fallbackToLargestRoot !== true ? viewportRoots : roots;
+  const root = [...candidates].sort(compareViewportRoots)[0];
+  if (!root) return undefined;
+  if (isPositiveFiniteRect(root.rect)) return { kind: 'reported', rect: root.rect };
+  return {
+    kind: 'missing',
+    reason:
+      root.rectStatus === 'invalid' || (root.rectStatus === undefined && root.rect !== undefined)
+        ? 'invalid'
+        : 'not-provided',
+  };
+}
+
+function isViewportRoot(root: IosSnapshotViewportRoot): boolean {
+  const type = normalizeType(root.type ?? '');
+  return type === 'application' || type === 'window';
+}
+
+function compareViewportRoots(
+  left: IosSnapshotViewportRoot,
+  right: IosSnapshotViewportRoot,
+): number {
+  const status = rootGeometryRank(right.rectStatus) - rootGeometryRank(left.rectStatus);
+  return status || rectArea(right.rect) - rectArea(left.rect);
+}
+
+function rootGeometryRank(status: IosSnapshotViewportRoot['rectStatus']): number {
+  return status === 'reported' ? 2 : status === 'invalid' ? 1 : 0;
+}
+
+function rectArea(rect: Rect | undefined): number {
+  return rect && isPositiveFiniteRect(rect) ? rect.width * rect.height : 0;
+}
+
 export function createIosSnapshotEngine(options: IosSnapshotEngineOptions = {}): IosSnapshotEngine {
   const foldPolicy = options.foldPolicy ?? DEFAULT_FOLD_POLICY;
   return Object.freeze({
@@ -46,13 +95,12 @@ export function publishIosSnapshot(
     input.stage === 'presented'
       ? input.validation.presentationKey
       : buildIosSnapshotPresentationKey(request);
+  const truncated =
+    input.stage === 'acquired' ? input.acquisition.truncated : input.presentation.payload.truncated;
   return {
     payload: {
       nodes: attachRefs(presentation.nodes),
-      truncated:
-        input.stage === 'acquired'
-          ? input.acquisition.truncated
-          : input.presentation.payload.truncated,
+      ...(truncated === undefined ? {} : { truncated }),
     },
     presentationKey,
     comparisonIdentity: buildIosSnapshotComparisonIdentity(input, request),

packages/capture-kit/src/ios-snapshot-engine/geometry.ts

@@ -107,17 +107,21 @@ function appendFoldedNode(
 
   const index = kept.length;
   keptDepth += 1;
+  const { hittable: sourceHittable, ...sourceNode } = node;
   kept.push({
     raw: {
-      ...node,
+      ...sourceNode,
       index,
       depth: keptDepth,
       parentIndex: keptIndex,
-      hittable:
-        node.parentIndex !== undefined &&
-        options.hittabilityAvailable !== false &&
-        node.hittable === true &&
-        isGeometricallyActionable(node.enabled !== false, decision.effectiveRect, viewport),
+      ...foldedHittability(
+        sourceHittable,
+        node.parentIndex !== undefined,
+        decision.effectiveRect,
+        node.enabled !== false,
+        viewport,
+        options,
+      ),
     },
     sourceIndex: node.index,
     ...(decision.effectiveRect ? { effectiveRect: decision.effectiveRect } : {}),
@@ -126,6 +130,25 @@ function appendFoldedNode(
   return { keptIndex, keptDepth };
 }
 
+function foldedHittability(
+  sourceHittable: RawSnapshotNode['hittable'],
+  hasParent: boolean,
+  effectiveRect: Rect | undefined,
+  enabled: boolean,
+  viewport: Rect,
+  options: IosSnapshotFoldOptions,
+): Partial<Pick<RawSnapshotNode, 'hittable'>> {
+  if (options.hittabilityAvailable === false) {
+    return sourceHittable === false ? { hittable: false } : {};
+  }
+  return {
+    hittable:
+      hasParent &&
+      sourceHittable === true &&
+      isGeometricallyActionable(enabled, effectiveRect, viewport),
+  };
+}
+
 function nextScrollAnchor(
   decision: GeometryDecision,
   parentAnchor: BranchState['anchor'],

packages/capture-kit/src/ios-snapshot-engine/index.ts

@@ -3,6 +3,7 @@ export {
   createIosSnapshotEngine,
   presentIosSnapshot,
   publishIosSnapshot,
+  resolveIosViewportEvidenceFromRoots,
 } from './engine.ts';
 export { presentIosRunnerSnapshot } from './runner-presentation.ts';
 export {
@@ -11,5 +12,5 @@ export {
 } from './semantic-index.ts';
 export { collectIosStructuralIdentifierSuppression } from './noise-structural.ts';
 export { findNearestScrollableContainer, mergeReplacement, updateReplacement } from './tree.ts';
-export { IosSnapshotEngineError } from './types.ts';
+export { IosSnapshotEngineError, toIosSnapshotEngineErrorDetails } from './types.ts';
 export type { SnapshotTreeRuleContext } from './tree.ts';

packages/capture-kit/src/ios-snapshot-engine/projection.ts

@@ -142,17 +142,19 @@ function createRegularProjectedNode(
   if (!isEligibleForIosRegularPresentation(node.raw)) return undefined;
   const depth = parent ? (parent.depth ?? 0) + 1 : 0;
   if (maximumDepth !== null && depth > maximumDepth) return undefined;
+  const hittable = isProjectedNodeHittable(node);
   return {
     ...node.raw,
     index,
     depth,
     parentIndex: parent?.index,
     ...(node.effectiveRect ? { rect: node.effectiveRect } : { rect: undefined }),
-    hittable: isProjectedNodeHittable(node),
+    ...(hittable === undefined ? {} : { hittable }),
   };
 }
 
-function isProjectedNodeHittable(node: IosSnapshotPresentationNode): boolean {
+function isProjectedNodeHittable(node: IosSnapshotPresentationNode): RawSnapshotNode['hittable'] {
+  if (node.raw.hittable === undefined) return undefined;
   return Boolean(
     node.raw.hittable === true &&
     node.effectiveRect &&

packages/capture-kit/src/ios-snapshot-engine/types.ts

@@ -64,3 +64,13 @@ export class IosSnapshotEngineError extends Error {
     this.details = details;
   }
 }
+
+export type IosSnapshotEnginePublicErrorDetails = Readonly<
+  IosSnapshotEngineFailureDetails & { reason: IosSnapshotEngineFailureReason }
+>;
+
+export function toIosSnapshotEngineErrorDetails(
+  error: IosSnapshotEngineError,
+): IosSnapshotEnginePublicErrorDetails {
+  return { reason: error.reason, ...error.details };
+}