diff --git a/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/src/commands/interaction/runtime/__tests__/test-utils/index.ts index 64396402e6..99552f78cb 100644 --- a/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -295,7 +295,8 @@ export function nonTouchableGroupSnapshot(): SnapshotState { return makeSnapshotState([ { index: 0, - depth: 0, + depth: 1, + parentIndex: 2, type: 'XCUIElementTypeOther', label: 'Clickable group', rect: { x: 10, y: 20, width: 300, height: 80 }, @@ -303,13 +304,20 @@ export function nonTouchableGroupSnapshot(): SnapshotState { }, { index: 1, - depth: 1, + depth: 2, parentIndex: 0, type: 'XCUIElementTypeOther', label: 'Decorative group', rect: { x: 30, y: 40, width: 60, height: 20 }, hittable: false, }, + { + index: 2, + depth: 0, + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: true, + }, ]); } diff --git a/src/core/interaction-targeting.fixtures.ts b/src/core/interaction-targeting.fixtures.ts index 1f88de3d29..046e4ab502 100644 --- a/src/core/interaction-targeting.fixtures.ts +++ b/src/core/interaction-targeting.fixtures.ts @@ -3,7 +3,8 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; export const EQUIVALENT_WRAPPER_CHAIN_NODES: RawSnapshotNode[] = [ { index: 0, - depth: 0, + depth: 1, + parentIndex: 3, type: 'XCUIElementTypeCell', label: 'Chat', rect: { x: 10, y: 20, width: 300, height: 60 }, @@ -27,6 +28,13 @@ export const EQUIVALENT_WRAPPER_CHAIN_NODES: RawSnapshotNode[] = [ rect: { x: 24, y: 32, width: 80, height: 20 }, hittable: false, }, + { + index: 3, + depth: 0, + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: true, + }, ]; export const ELEMENT14_DISTINCT_SUBTREE_NODES: RawSnapshotNode[] = [ diff --git a/src/core/interaction-targeting.test.ts b/src/core/interaction-targeting.test.ts index 5e544e20f9..1f88b40a4d 100644 --- a/src/core/interaction-targeting.test.ts +++ b/src/core/interaction-targeting.test.ts @@ -21,7 +21,10 @@ import { test('collapses one same-label wrapper chain to its shared actionable node', () => { const snapshot = makeSnapshotState(EQUIVALENT_WRAPPER_CHAIN_NODES); - const result = classifyActionableTouchCandidates(snapshot.nodes, snapshot.nodes); + const result = classifyActionableTouchCandidates( + snapshot.nodes, + snapshot.nodes.filter((node) => node.label === 'Chat'), + ); assert.equal(result.kind, 'equivalent'); if (result.kind === 'equivalent') assert.equal(result.node.index, 1); @@ -46,7 +49,8 @@ test('promotes static text inside a hittable row to the row', () => { const snapshot = makeSnapshotState([ { index: 0, - depth: 0, + depth: 1, + parentIndex: 2, type: 'XCUIElementTypeCell', label: 'Account row', rect: { x: 10, y: 20, width: 300, height: 60 }, @@ -54,13 +58,20 @@ test('promotes static text inside a hittable row to the row', () => { }, { index: 1, - depth: 1, + depth: 2, parentIndex: 0, type: 'XCUIElementTypeStaticText', label: 'Account', rect: { x: 24, y: 32, width: 80, height: 20 }, hittable: false, }, + { + index: 2, + depth: 0, + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: true, + }, ]); const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[1]!); @@ -187,6 +198,85 @@ test('prevents full-screen window-like ancestors from stealing taps', () => { assert.equal(resolution.node.label, 'Status'); }); +test('prevents a full-screen ancestor from stealing taps in a rootless Android-shaped tree', () => { + const snapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'android.widget.FrameLayout', + rect: { x: 0, y: 0, width: 1080, height: 2400 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'android.widget.TextView', + label: 'Inbox', + rect: { x: 24, y: 200, width: 160, height: 64 }, + hittable: false, + }, + ]); + + const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[1]!); + + assert.equal(resolution.reason, 'overly-broad-ancestor'); + assert.equal(resolution.node.label, 'Inbox'); +}); + +test('applies the rootless viewport fallback without platform-shaped node types', () => { + const snapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Container', + rect: { x: 0, y: 0, width: 1080, height: 2400 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Label', + label: 'Inbox', + rect: { x: 24, y: 200, width: 160, height: 64 }, + hittable: false, + }, + ]); + + const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[1]!); + + assert.equal(resolution.reason, 'overly-broad-ancestor'); + assert.equal(resolution.node.label, 'Inbox'); +}); + +test('keeps a rootless ancestor when its rectangle matches the target', () => { + const targetRect = { x: 24, y: 200, width: 160, height: 64 }; + const snapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Container', + rect: targetRect, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Label', + label: 'Inbox', + rect: targetRect, + hittable: false, + }, + ]); + + const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[1]!); + + assert.equal(resolution.reason, 'hittable-ancestor'); + assert.equal(resolution.node.index, 0); +}); + test('falls back to the original node when no usable touch target exists', () => { const snapshot = makeSnapshotState([ { diff --git a/src/core/interaction-targeting.ts b/src/core/interaction-targeting.ts index fe0d80cce7..ffa91df1e1 100644 --- a/src/core/interaction-targeting.ts +++ b/src/core/interaction-targeting.ts @@ -1,11 +1,10 @@ import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; -import { centerOfRect } from '@agent-device/kernel/snapshot'; -import { containsPoint, pickLargestRect } from '@agent-device/kernel/rect'; import { findNearestAncestor, findSnapshotAncestor, normalizeType, isViewportRootNode, + resolveViewportRect, } from '@agent-device/contracts/snapshot'; import { isSnapshotNodeInteractionBlocked } from '@agent-device/capture-kit/snapshot-occlusion'; import { @@ -223,7 +222,7 @@ function isOverlyBroadAncestor( if (isScrollingContainer(ancestor) && !areRectsApproximatelyEqual(nodeRect, ancestorRect)) { return true; } - const rootViewportRect = resolveRootViewportRect(nodes, nodeRect, index); + const rootViewportRect = resolveViewportRect(nodes, nodeRect, index?.viewportRootRects); if (!rootViewportRect) return false; if (!isRectViewportSized(ancestorRect, rootViewportRect)) return false; return !areRectsApproximatelyEqual(nodeRect, ancestorRect); @@ -243,26 +242,6 @@ function isScrollingContainer(node: SnapshotNode): boolean { ); } -function resolveRootViewportRect( - nodes: SnapshotNode[], - targetRect: Rect, - index: ActionableTouchIndex | undefined, -): Rect | null { - const targetCenter = centerOfRect(targetRect); - const viewportRects = - index?.viewportRootRects ?? - nodes - .filter(isViewportRootNode) - .map((node) => normalizeRect(node.rect)) - .filter((rect): rect is Rect => rect !== null); - if (viewportRects.length === 0) return null; - - const containingRects = viewportRects.filter((rect) => - containsPoint(rect, targetCenter.x, targetCenter.y), - ); - return pickLargestRect(containingRects.length > 0 ? containingRects : viewportRects); -} - function buildActionableTouchIndex(nodes: readonly SnapshotNode[]): ActionableTouchIndex { const nodesByIndex = new Map(); const childrenByParentIndex = new Map(); diff --git a/src/core/selector-pipeline.test.ts b/src/core/selector-pipeline.test.ts index 506b3e0ee2..340939c7a9 100644 --- a/src/core/selector-pipeline.test.ts +++ b/src/core/selector-pipeline.test.ts @@ -63,7 +63,8 @@ const COVERED_TREE: RawSnapshotNode[] = [ const PROMOTABLE_TREE: RawSnapshotNode[] = [ { index: 0, - depth: 0, + depth: 1, + parentIndex: 2, type: 'XCUIElementTypeCell', label: 'Account row', rect: { x: 10, y: 20, width: 300, height: 60 }, @@ -71,13 +72,20 @@ const PROMOTABLE_TREE: RawSnapshotNode[] = [ }, { index: 1, - depth: 1, + depth: 2, parentIndex: 0, type: 'XCUIElementTypeStaticText', label: 'Account', rect: { x: 24, y: 32, width: 80, height: 20 }, hittable: false, }, + { + index: 2, + depth: 0, + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: true, + }, ]; /** diff --git a/src/daemon/handlers/__tests__/interaction-touch-press.test.ts b/src/daemon/handlers/__tests__/interaction-touch-press.test.ts index 089ed845f7..11321eb39f 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-press.test.ts +++ b/src/daemon/handlers/__tests__/interaction-touch-press.test.ts @@ -136,6 +136,7 @@ test('press @ref promotes a non-hittable node to its hittable ancestor before ta nodes: attachRefs([ { index: 0, + parentIndex: 2, type: 'XCUIElementTypeCell', label: 'Settings row', rect: { x: 20, y: 100, width: 320, height: 72 }, @@ -151,6 +152,12 @@ test('press @ref promotes a non-hittable node to its hittable ancestor before ta enabled: false, hittable: false, }, + { + index: 2, + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + hittable: true, + }, ]), createdAt: Date.now(), backend: 'xctest',