Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions src/commands/interaction/runtime/__tests__/test-utils/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -295,21 +295,29 @@ export function nonTouchableGroupSnapshot(): SnapshotState {
return makeSnapshotState([
{
index: 0,
depth: 0,
depth: 1,
parentIndex: 2,
type: 'XCUIElementTypeOther',
label: 'Clickable group',
rect: { x: 10, y: 20, width: 300, height: 80 },
hittable: true,
},
{
index: 1,
depth: 1,
depth: 2,
parentIndex: 0,
type: 'XCUIElementTypeOther',
label: 'Decorative group',
rect: { x: 30, y: 40, width: 60, height: 20 },
hittable: false,
},
{
index: 2,
depth: 0,
type: 'XCUIElementTypeApplication',
rect: { x: 0, y: 0, width: 390, height: 844 },
hittable: true,
},
]);
}

Expand Down
10 changes: 9 additions & 1 deletion src/core/interaction-targeting.fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
export const EQUIVALENT_WRAPPER_CHAIN_NODES: RawSnapshotNode[] = [
{
index: 0,
depth: 0,
depth: 1,
parentIndex: 3,
type: 'XCUIElementTypeCell',
label: 'Chat',
rect: { x: 10, y: 20, width: 300, height: 60 },
Expand All @@ -27,6 +28,13 @@ export const EQUIVALENT_WRAPPER_CHAIN_NODES: RawSnapshotNode[] = [
rect: { x: 24, y: 32, width: 80, height: 20 },
hittable: false,
},
{
index: 3,
depth: 0,
type: 'XCUIElementTypeApplication',
rect: { x: 0, y: 0, width: 390, height: 844 },
hittable: true,
},
];

export const ELEMENT14_DISTINCT_SUBTREE_NODES: RawSnapshotNode[] = [
Expand Down
96 changes: 93 additions & 3 deletions src/core/interaction-targeting.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@ import {
test('collapses one same-label wrapper chain to its shared actionable node', () => {
const snapshot = makeSnapshotState(EQUIVALENT_WRAPPER_CHAIN_NODES);

const result = classifyActionableTouchCandidates(snapshot.nodes, snapshot.nodes);
const result = classifyActionableTouchCandidates(
snapshot.nodes,
snapshot.nodes.filter((node) => node.label === 'Chat'),
);

assert.equal(result.kind, 'equivalent');
if (result.kind === 'equivalent') assert.equal(result.node.index, 1);
Expand All @@ -46,21 +49,29 @@ test('promotes static text inside a hittable row to the row', () => {
const snapshot = makeSnapshotState([
{
index: 0,
depth: 0,
depth: 1,
parentIndex: 2,
type: 'XCUIElementTypeCell',
label: 'Account row',
rect: { x: 10, y: 20, width: 300, height: 60 },
hittable: true,
},
{
index: 1,
depth: 1,
depth: 2,
parentIndex: 0,
type: 'XCUIElementTypeStaticText',
label: 'Account',
rect: { x: 24, y: 32, width: 80, height: 20 },
hittable: false,
},
{
index: 2,
depth: 0,
type: 'XCUIElementTypeApplication',
rect: { x: 0, y: 0, width: 390, height: 844 },
hittable: true,
},
]);

const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[1]!);
Expand Down Expand Up @@ -187,6 +198,85 @@ test('prevents full-screen window-like ancestors from stealing taps', () => {
assert.equal(resolution.node.label, 'Status');
});

test('prevents a full-screen ancestor from stealing taps in a rootless Android-shaped tree', () => {
const snapshot = makeSnapshotState([
{
index: 0,
depth: 0,
type: 'android.widget.FrameLayout',
rect: { x: 0, y: 0, width: 1080, height: 2400 },
hittable: true,
},
{
index: 1,
depth: 1,
parentIndex: 0,
type: 'android.widget.TextView',
label: 'Inbox',
rect: { x: 24, y: 200, width: 160, height: 64 },
hittable: false,
},
]);

const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[1]!);

assert.equal(resolution.reason, 'overly-broad-ancestor');
assert.equal(resolution.node.label, 'Inbox');
});

test('applies the rootless viewport fallback without platform-shaped node types', () => {
const snapshot = makeSnapshotState([
{
index: 0,
depth: 0,
type: 'Container',
rect: { x: 0, y: 0, width: 1080, height: 2400 },
hittable: true,
},
{
index: 1,
depth: 1,
parentIndex: 0,
type: 'Label',
label: 'Inbox',
rect: { x: 24, y: 200, width: 160, height: 64 },
hittable: false,
},
]);

const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[1]!);

assert.equal(resolution.reason, 'overly-broad-ancestor');
assert.equal(resolution.node.label, 'Inbox');
});

test('keeps a rootless ancestor when its rectangle matches the target', () => {
const targetRect = { x: 24, y: 200, width: 160, height: 64 };
const snapshot = makeSnapshotState([
{
index: 0,
depth: 0,
type: 'Container',
rect: targetRect,
hittable: true,
},
{
index: 1,
depth: 1,
parentIndex: 0,
type: 'Label',
label: 'Inbox',
rect: targetRect,
hittable: false,
},
]);

const resolution = resolveActionableTouchResolution(snapshot.nodes, snapshot.nodes[1]!);

assert.equal(resolution.reason, 'hittable-ancestor');
assert.equal(resolution.node.index, 0);
});

test('falls back to the original node when no usable touch target exists', () => {
const snapshot = makeSnapshotState([
{
Expand Down
25 changes: 2 additions & 23 deletions src/core/interaction-targeting.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot';
import { centerOfRect } from '@agent-device/kernel/snapshot';
import { containsPoint, pickLargestRect } from '@agent-device/kernel/rect';
import {
findNearestAncestor,
findSnapshotAncestor,
normalizeType,
isViewportRootNode,
resolveViewportRect,
} from '@agent-device/contracts/snapshot';
import { isSnapshotNodeInteractionBlocked } from '@agent-device/capture-kit/snapshot-occlusion';
import {
Expand Down Expand Up @@ -223,7 +222,7 @@ function isOverlyBroadAncestor(
if (isScrollingContainer(ancestor) && !areRectsApproximatelyEqual(nodeRect, ancestorRect)) {
return true;
}
const rootViewportRect = resolveRootViewportRect(nodes, nodeRect, index);
const rootViewportRect = resolveViewportRect(nodes, nodeRect, index?.viewportRootRects);
if (!rootViewportRect) return false;
if (!isRectViewportSized(ancestorRect, rootViewportRect)) return false;
return !areRectsApproximatelyEqual(nodeRect, ancestorRect);
Expand All @@ -243,26 +242,6 @@ function isScrollingContainer(node: SnapshotNode): boolean {
);
}

function resolveRootViewportRect(
nodes: SnapshotNode[],
targetRect: Rect,
index: ActionableTouchIndex | undefined,
): Rect | null {
const targetCenter = centerOfRect(targetRect);
const viewportRects =
index?.viewportRootRects ??
nodes
.filter(isViewportRootNode)
.map((node) => normalizeRect(node.rect))
.filter((rect): rect is Rect => rect !== null);
if (viewportRects.length === 0) return null;

const containingRects = viewportRects.filter((rect) =>
containsPoint(rect, targetCenter.x, targetCenter.y),
);
return pickLargestRect(containingRects.length > 0 ? containingRects : viewportRects);
}

function buildActionableTouchIndex(nodes: readonly SnapshotNode[]): ActionableTouchIndex {
const nodesByIndex = new Map<number, SnapshotNode>();
const childrenByParentIndex = new Map<number, SnapshotNode[]>();
Expand Down
12 changes: 10 additions & 2 deletions src/core/selector-pipeline.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,21 +63,29 @@ const COVERED_TREE: RawSnapshotNode[] = [
const PROMOTABLE_TREE: RawSnapshotNode[] = [
{
index: 0,
depth: 0,
depth: 1,
parentIndex: 2,
type: 'XCUIElementTypeCell',
label: 'Account row',
rect: { x: 10, y: 20, width: 300, height: 60 },
hittable: true,
},
{
index: 1,
depth: 1,
depth: 2,
parentIndex: 0,
type: 'XCUIElementTypeStaticText',
label: 'Account',
rect: { x: 24, y: 32, width: 80, height: 20 },
hittable: false,
},
{
index: 2,
depth: 0,
type: 'XCUIElementTypeApplication',
rect: { x: 0, y: 0, width: 390, height: 844 },
hittable: true,
},
];

/**
Expand Down
7 changes: 7 additions & 0 deletions src/daemon/handlers/__tests__/interaction-touch-press.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ test('press @ref promotes a non-hittable node to its hittable ancestor before ta
nodes: attachRefs([
{
index: 0,
parentIndex: 2,
type: 'XCUIElementTypeCell',
label: 'Settings row',
rect: { x: 20, y: 100, width: 320, height: 72 },
Expand All @@ -151,6 +152,12 @@ test('press @ref promotes a non-hittable node to its hittable ancestor before ta
enabled: false,
hittable: false,
},
{
index: 2,
type: 'XCUIElementTypeApplication',
rect: { x: 0, y: 0, width: 390, height: 844 },
hittable: true,
},
]),
createdAt: Date.now(),
backend: 'xctest',
Expand Down
Loading