From d9f1d2106e660abb88e3026270457880d131716f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 2 Sep 2026 19:25:40 +0200 Subject: [PATCH 1/2] feat: add strict native absence assertion --- docs/adr/0012-interactive-replay.md | 6 +- examples/test-app/README.md | 2 +- packages/contracts/package.json | 4 + .../contracts/src/client-selector-read.ts | 5 +- packages/contracts/src/interaction-error.ts | 1 + packages/contracts/src/is-predicate.ts | 13 + packages/selectors/src/index.ts | 1 + packages/selectors/src/internal/predicates.ts | 23 +- .../src/internal/resolution-policy.ts | 4 +- scripts/layering/package-boundaries.test.ts | 1 + src/__tests__/cli-grammar.test.ts | 2 +- .../is-argument-surface-parity.test.ts | 71 ++++- src/cli-schema/cli-help-topics.test.ts | 2 +- src/cli-schema/cli-help.ts | 2 +- src/commands/interaction/metadata.ts | 7 +- .../runtime/selector-read-shared.ts | 38 ++- .../interaction/runtime/selector-read.test.ts | 256 ++++++++++++++++++ .../interaction/runtime/selector-read.ts | 137 ++++++---- src/commands/interaction/selectors.ts | 8 + src/core/absence-observation-errors.ts | 76 ++++++ src/core/absence-observation-resolution.ts | 47 ++++ src/core/absence-observation.test.ts | 94 +++++++ src/core/absence-observation.ts | 111 ++++++++ src/core/selector-pipeline-policy.ts | 4 +- src/daemon/__tests__/is-runtime.test.ts | 107 +++++++- .../interaction-target-evidence.test.ts | 31 +++ ...sion-replay-dispatch-selector-miss.test.ts | 49 ++++ src/daemon/selector-capture-runtime.ts | 8 +- src/daemon/selector-runtime.ts | 12 + website/docs/docs/commands.md | 8 +- 30 files changed, 1029 insertions(+), 101 deletions(-) create mode 100644 packages/contracts/src/is-predicate.ts create mode 100644 src/core/absence-observation-errors.ts create mode 100644 src/core/absence-observation-resolution.ts create mode 100644 src/core/absence-observation.test.ts create mode 100644 src/core/absence-observation.ts diff --git a/docs/adr/0012-interactive-replay.md b/docs/adr/0012-interactive-replay.md index be6568d88e..96218d4825 100644 --- a/docs/adr/0012-interactive-replay.md +++ b/docs/adr/0012-interactive-replay.md @@ -279,14 +279,16 @@ A recorded `id` never matches a node without that id. > verify against the live tree's real value at replay time, so the evidence is dropped rather than > published unverified. See ADR 0017's session-scoped echo protection amendment for the mechanism. > - **`get` — unchanged**; already covered by the pre-dispatch path and the post-resolution guard. -> - **`is` (all predicates except `exists`) — covered, `pre-dispatch`, the `get` pattern end-to-end.** +> - **`is` (all predicates except `exists` and `absent`) — covered, `pre-dispatch`, the `get` pattern end-to-end.** > `is` resolves a unique node immediately, so pre-action verification is semantically valid; the > resolved node/tree feed record-time evidence, and dispatch threads `replayTargetGuard` into > `assertExpectedResolvedTarget` exactly like `get`. The direct-iOS `is`/`wait` fast paths are gated > off during recording and guarded replays, mirroring `get`'s existing recording gate. > - **Intentionally deferred, with tests proving no annotation is recorded and no identity check runs:** > `is exists` (existence assertion with no unique winner; wait-like semantics without the -> guard-critical role), every read-only `find` variant (fuzzy-locator resolution has no +> guard-critical role), `is absent` (a strict one-capture absence observation has no resolved +> winner; it records as an ordinary observation and its `predicate_failed` failure is always an +> action-failure, never an identity mismatch), every read-only `find` variant (fuzzy-locator resolution has no > selector-chain identity token for the classifier, and publication already refuses mutating `find` > as non-verifiable), and `wait text`/`wait stable`/duration waits/`wait @ref` (no element target, or > a session-local ref that ADR 0016 already refuses to publish; `wait @ref` is rejected rather than diff --git a/examples/test-app/README.md b/examples/test-app/README.md index 46735e168e..0439cab8f6 100644 --- a/examples/test-app/README.md +++ b/examples/test-app/README.md @@ -39,7 +39,7 @@ These are the main case families this app can support without adding more screen - `fill` on single-line and multiline fields - `type` after focus for append flows - `get text` on headings, badges, summaries, and accordion content -- `is visible` and `is exists` assertions +- `is visible`, `is exists`, and `is absent` assertions - `wait` for async loading and success states - `diff snapshot` after dismissals and submits - long-list scrolling and `scrollintoview` diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 9bb907f36b..9d3564a0ad 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -239,6 +239,10 @@ "types": "./src/interaction-error.ts", "default": "./src/interaction-error.ts" }, + "./is-predicate": { + "types": "./src/is-predicate.ts", + "default": "./src/is-predicate.ts" + }, "./interaction-guarantees": { "types": "./src/interaction-guarantees.ts", "default": "./src/interaction-guarantees.ts" diff --git a/packages/contracts/src/client-selector-read.ts b/packages/contracts/src/client-selector-read.ts index 7fc67a777f..6a7f1e0854 100644 --- a/packages/contracts/src/client-selector-read.ts +++ b/packages/contracts/src/client-selector-read.ts @@ -7,6 +7,7 @@ import type { } from './client-capture.ts'; import type { DeviceCommandBaseOptions } from './client-connection.ts'; import type { ElementTarget } from './client-target.ts'; +import type { IsPredicate } from './is-predicate.ts'; /** * #1271 stage 2 (ADR 0012 amendment): `get`/`is`/`find` are observation-only @@ -29,7 +30,7 @@ export type GetOptions = DeviceCommandBaseOptions & export type IsTextPredicateOptions = DeviceCommandBaseOptions & SelectorSnapshotCommandOptions & RecordControlOptions & { - predicate: 'text'; + predicate: Extract; selector: string; value: string; }; @@ -37,7 +38,7 @@ export type IsTextPredicateOptions = DeviceCommandBaseOptions & export type IsStatePredicateOptions = DeviceCommandBaseOptions & SelectorSnapshotCommandOptions & RecordControlOptions & { - predicate: 'visible' | 'hidden' | 'exists' | 'editable' | 'selected' | 'focused'; + predicate: Exclude; selector: string; value?: never; }; diff --git a/packages/contracts/src/interaction-error.ts b/packages/contracts/src/interaction-error.ts index 324df4dcca..5d136f2a6d 100644 --- a/packages/contracts/src/interaction-error.ts +++ b/packages/contracts/src/interaction-error.ts @@ -1,4 +1,5 @@ /** Machine-readable `error.details.reason` values shared by interaction producers and adapters. */ export const INTERACTION_ERROR_REASONS = { selectorNotFound: 'selector_not_found', + predicateFailed: 'predicate_failed', } as const; diff --git a/packages/contracts/src/is-predicate.ts b/packages/contracts/src/is-predicate.ts new file mode 100644 index 0000000000..217d960c3b --- /dev/null +++ b/packages/contracts/src/is-predicate.ts @@ -0,0 +1,13 @@ +/** The complete predicate vocabulary accepted by the `is` command. */ +export const IS_PREDICATES = [ + 'visible', + 'hidden', + 'exists', + 'absent', + 'editable', + 'selected', + 'focused', + 'text', +] as const; + +export type IsPredicate = (typeof IS_PREDICATES)[number]; diff --git a/packages/selectors/src/index.ts b/packages/selectors/src/index.ts index c49b28bce8..0b33e4f820 100644 --- a/packages/selectors/src/index.ts +++ b/packages/selectors/src/index.ts @@ -44,6 +44,7 @@ import { } from './internal/replay.ts'; export type { FindAction, FindLocator } from './internal/find.ts'; +export { IS_PREDICATES } from '@agent-device/contracts/is-predicate'; export type { IsPredicate } from './internal/predicates.ts'; export type { PolicyResolutionOutcome, diff --git a/packages/selectors/src/internal/predicates.ts b/packages/selectors/src/internal/predicates.ts index 346dc028d5..4e66ae8310 100644 --- a/packages/selectors/src/internal/predicates.ts +++ b/packages/selectors/src/internal/predicates.ts @@ -1,5 +1,6 @@ import { refuse, type SelectorArgumentRefusal } from './argument-refusal.ts'; +import { IS_PREDICATES, type IsPredicate } from '@agent-device/contracts/is-predicate'; import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; import type { SnapshotState } from '@agent-device/kernel/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; @@ -12,23 +13,15 @@ import { import { isNodeEditable, isNodeVisible } from './node.ts'; import { tryParseSelectorChain } from './parse.ts'; -export type IsPredicate = - | 'visible' - | 'hidden' - | 'exists' - | 'editable' - | 'selected' - | 'focused' - | 'text'; +export type { IsPredicate } from '@agent-device/contracts/is-predicate'; // Module-private since `checkIsPredicate` became the admission API: a caller that tests the // vocabulary without going through admission is how the case-normalization drift started. function isSupportedPredicate(input: string): input is IsPredicate { - return ['visible', 'hidden', 'exists', 'editable', 'selected', 'focused', 'text'].includes(input); + return (IS_PREDICATES as readonly string[]).includes(input); } -export const IS_PREDICATE_REQUIRED_MESSAGE = - 'is requires predicate: visible|hidden|exists|editable|selected|focused|text'; +export const IS_PREDICATE_REQUIRED_MESSAGE = `is requires predicate: ${IS_PREDICATES.join('|')}`; /** * The one `is` predicate admission check. Three call sites used to state this rule @@ -67,7 +60,7 @@ export function normalizeIsPositionals(positionals: string[]): string[] { } export function evaluateIsPredicate(params: { - predicate: Exclude; + predicate: Exclude; node: SnapshotState['nodes'][number]; nodes: SnapshotState['nodes']; expectedText?: string; @@ -102,6 +95,8 @@ export function evaluateIsPredicate(params: { case 'text': pass = actualText === (expectedText ?? ''); break; + default: + return assertNever(predicate); } const details = predicate === 'text' @@ -115,6 +110,10 @@ export function evaluateIsPredicate(params: { return { pass, actualText, details }; } +function assertNever(value: never): never { + throw new Error(`Unhandled is predicate: ${String(value)}`); +} + function isAssertionVisible( node: SnapshotState['nodes'][number], visibility: SnapshotVisibility, diff --git a/packages/selectors/src/internal/resolution-policy.ts b/packages/selectors/src/internal/resolution-policy.ts index 573c6603fe..2f766246e2 100644 --- a/packages/selectors/src/internal/resolution-policy.ts +++ b/packages/selectors/src/internal/resolution-policy.ts @@ -61,12 +61,12 @@ export const SELECTOR_RESOLUTION_POLICIES = { ambiguity: 'disambiguate', requireRect: false, }, - /** `is` non-exists predicates and `get attrs` — ties reject, never guess. */ + /** `is` predicates other than `exists`/`absent`, and `get attrs` — ties reject, never guess. */ readUnique: { ambiguity: 'fail-closed', requireRect: false, }, - /** `exists` and find's read-only actions — presence is the question. */ + /** `exists`/`absent` and find's read-only actions — presence is the question. */ readAny: { ambiguity: 'first-match', requireRect: false, diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 8820b2fbec..ad92488412 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -108,6 +108,7 @@ const CONTRACT_EXPORTS = [ '@agent-device/contracts/interactor-operation-catalog', '@agent-device/contracts/interactor-types', '@agent-device/contracts/ios-snapshot', + '@agent-device/contracts/is-predicate', '@agent-device/contracts/keyboard', '@agent-device/contracts/keyboard-runtime', '@agent-device/contracts/local-interactor-operation-set', diff --git a/src/__tests__/cli-grammar.test.ts b/src/__tests__/cli-grammar.test.ts index 221ee66bec..fdc71109b7 100644 --- a/src/__tests__/cli-grammar.test.ts +++ b/src/__tests__/cli-grammar.test.ts @@ -169,7 +169,7 @@ test('is grammar explains the predicate/selector-key collision on invalid predic assert.equal(err.code, 'INVALID_ARGS'); assert.match( err.message, - /is requires predicate: visible\|hidden\|exists\|editable\|selected\|focused\|text/, + /is requires predicate: visible\|hidden\|exists\|absent\|editable\|selected\|focused\|text/, ); assert.match(err.details?.hint ?? '', /is /); assert.match(err.details?.hint ?? '', /visible=true/); diff --git a/src/__tests__/is-argument-surface-parity.test.ts b/src/__tests__/is-argument-surface-parity.test.ts index 81f48c5927..f924638a17 100644 --- a/src/__tests__/is-argument-surface-parity.test.ts +++ b/src/__tests__/is-argument-surface-parity.test.ts @@ -1,7 +1,15 @@ import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; import { test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; -import { checkIsArgs, checkIsPredicate, IS_PREDICATE_USAGE_HINT } from '@agent-device/selectors'; +import { + IS_PREDICATES, + checkIsArgs, + checkIsPredicate, + IS_PREDICATE_USAGE_HINT, +} from '@agent-device/selectors'; +import { interactionCommandMetadata } from '../commands/interaction/metadata.ts'; import { readInputFromCli } from '../commands/cli-grammar.ts'; import type { CliFlags } from '@agent-device/contracts/command'; @@ -19,6 +27,16 @@ import type { CliFlags } from '@agent-device/contracts/command'; // helper-only test could not catch. const BASE_FLAGS = {} as CliFlags; +const EXPECTED_IS_PREDICATES = [ + 'visible', + 'hidden', + 'exists', + 'absent', + 'editable', + 'selected', + 'focused', + 'text', +] as const; type Verdict = { ok: true; predicate: string } | { ok: false; message: string; hint?: string }; @@ -77,6 +95,18 @@ const CASES: readonly { expect: 'accept', predicate: 'text', }, + { + name: 'absence predicate first', + positionals: ['absent', 'id="gone"'], + expect: 'accept', + predicate: 'absent', + }, + { + name: 'absence predicate after selector', + positionals: ['id="gone"', 'absent'], + expect: 'accept', + predicate: 'absent', + }, { name: 'unknown predicate', positionals: ['shiny', 'id="ok"'], expect: 'refuse' }, { name: 'no predicate at all', positionals: [], expect: 'refuse' }, ]; @@ -120,3 +150,42 @@ test('an unsupported predicate is refused with the same message and hint everywh assert.equal(daemon.hint, IS_PREDICATE_USAGE_HINT); assert.equal(cli.hint, IS_PREDICATE_USAGE_HINT); }); + +test('command docs list every is predicate', () => { + const docs = fs.readFileSync( + path.resolve(import.meta.dirname, '../..', 'website/docs/docs/commands.md'), + 'utf8', + ); + const predicateList = docs.match(/Supported predicates are ([^.]+)\./)?.[1] ?? ''; + assert.deepEqual(IS_PREDICATES, EXPECTED_IS_PREDICATES); + for (const predicate of EXPECTED_IS_PREDICATES) { + assert.match(predicateList, new RegExp(`\\b${predicate}\\b`)); + } + + const metadata = interactionCommandMetadata.find((entry) => entry.name === 'is'); + const predicateSchema = metadata?.inputSchema.properties?.predicate; + assert.deepEqual(predicateSchema?.enum, IS_PREDICATES); +}); + +test('the CLI refuses scoped and depth-limited absence captures as INVALID_ARGS', () => { + for (const [flag, value] of [ + ['snapshotScope', 'Login'], + ['snapshotDepth', 2], + ] as const) { + assert.throws( + () => + readInputFromCli('is', ['absent', 'label="Gone"'], { + ...BASE_FLAGS, + [flag]: value, + } as CliFlags), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal(error.details?.command, 'is'); + assert.equal(error.details?.predicate, 'absent'); + assert.equal(error.details?.rejectedOption, flag === 'snapshotScope' ? 'scope' : 'depth'); + return true; + }, + ); + } +}); diff --git a/src/cli-schema/cli-help-topics.test.ts b/src/cli-schema/cli-help-topics.test.ts index b7fa2c6338..29288b8aee 100644 --- a/src/cli-schema/cli-help-topics.test.ts +++ b/src/cli-schema/cli-help-topics.test.ts @@ -373,7 +373,7 @@ test('usageForCommand resolves web help topic', async () => { assert.match(help, /agent-device screenshot \.\/artifacts\/web-home\.png --platform web/); assert.match(help, /agent-device close --platform web/); assert.match(help, /open , snapshot -i, get text\/attrs/); - assert.match(help, /is visible\/exists\/text, find text\/selector/); + assert.match(help, /is visible\/hidden\/exists\/absent\/focused\/text, find text\/selector/); assert.match(help, /click\/press @ref or selector/); assert.match(help, /network dump/); assert.match(help, /audio probe/); diff --git a/src/cli-schema/cli-help.ts b/src/cli-schema/cli-help.ts index 142a7d104c..4ce36f479c 100644 --- a/src/cli-schema/cli-help.ts +++ b/src/cli-schema/cli-help.ts @@ -917,7 +917,7 @@ First-slice loop: agent-device close --platform web Supported in agent-device web sessions: - open , snapshot -i, get text/attrs, is visible/exists/text, find text/selector, click/press @ref or selector, hover @ref or selector, fill/type @ref or selector, wait text/selector, network dump, audio probe, screenshot, record start/stop with WebM output, close, and replay scripts made from those commands. + open , snapshot -i, get text/attrs, is visible/hidden/exists/absent/focused/text, find text/selector, click/press @ref or selector, hover @ref or selector, fill/type @ref or selector, wait text/selector, network dump, audio probe, screenshot, record start/stop with WebM output, close, and replay scripts made from those commands. hover moves the pointer without pressing so hover-gated UI (row toolbars, menus) appears; use --settle to read what it revealed, then act on the fresh refs. hover @ref hovers the browser element handle directly; pair --settle with a selector or coordinates (web refs carry no geometry, as with click @ref --settle). Web only: touch platforms have no hover state, so hover-gated flows there need a different entry point. Out of scope for agent-device web support: diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 435eaea51f..b4b7f2bf35 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -11,6 +11,7 @@ import { readGesturePayload, } from '@agent-device/contracts/gesture-input'; import { SCROLL_DURATION_MAX_MS } from '@agent-device/contracts/scroll-command'; +import { IS_PREDICATES } from '@agent-device/contracts/is-predicate'; import { SCROLL_DIRECTIONS, SWIPE_PATTERNS, @@ -69,7 +70,7 @@ const interactionCommandDescriptions = { scroll: 'Scroll in a direction, or toward the top/bottom edge of scrollable content. The optional amount is the finger-path fraction of the viewport axis; app scroll physics determine the final content offset.', get: 'Read text or accessibility attributes from a snapshot ref or selector without changing the app. Use format text for visible content or attrs for the element attribute map.', - is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, editable, selected, focused, or text. Use wait when the condition may appear asynchronously.', + is: 'Check whether a selector satisfies a UI predicate such as visible, hidden, exists, absent, editable, selected, focused, or text. `absent` passes only when one readable, complete, unscoped, full-depth accessibility capture has zero matches. Use wait when the condition may appear asynchronously.', find: 'Find by text/label/value/role/id and run action', gesture: 'Perform a structured pan, fling, swipe, pinch, rotate, transform, or drag gesture. Select the gesture kind, then provide only the inputs that apply to that kind.', @@ -171,9 +172,7 @@ const getFields = { }; const isFields = { - predicate: requiredField( - enumField(['visible', 'hidden', 'exists', 'editable', 'selected', 'focused', 'text'] as const), - ), + predicate: requiredField(enumField(IS_PREDICATES)), selector: requiredField(stringField()), value: stringField(), ...selectorSnapshotFields(), diff --git a/src/commands/interaction/runtime/selector-read-shared.ts b/src/commands/interaction/runtime/selector-read-shared.ts index 324286dc57..09dc8bfb5b 100644 --- a/src/commands/interaction/runtime/selector-read-shared.ts +++ b/src/commands/interaction/runtime/selector-read-shared.ts @@ -3,6 +3,7 @@ import type { CommandContext, CommandSessionRecord, } from '../../../runtime-contract.ts'; +import type { BackendSnapshotResult } from '../../../backend.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { SnapshotNode, @@ -80,15 +81,7 @@ export async function captureSelectorSnapshot( ? { includeHiddenContentHints: captureOptions.includeHiddenContentHints } : {}), }); - const snapshot = - result.snapshot ?? - ({ - nodes: result.nodes ?? [], - truncated: result.truncated, - backend: result.backend as SnapshotState['backend'], - ...(result.quality ? { snapshotQuality: result.quality } : {}), - createdAt: now(runtime), - } satisfies SnapshotState); + const snapshot = snapshotStateFromResult(result, runtime); (options.signal ?? runtime.signal)?.throwIfAborted(); if ( captureOptions.updateSession && @@ -100,6 +93,33 @@ export async function captureSelectorSnapshot( return { sessionName, session, snapshot }; } +function snapshotStateFromResult( + result: BackendSnapshotResult, + runtime: AgentDeviceRuntime, +): SnapshotState { + if (result.snapshot) return mergeSnapshotAnnotations(result.snapshot, result); + return { + nodes: result.nodes ?? [], + truncated: result.truncated, + backend: result.backend as SnapshotState['backend'], + ...(result.quality ? { snapshotQuality: result.quality } : {}), + createdAt: now(runtime), + } satisfies SnapshotState; +} + +function mergeSnapshotAnnotations( + snapshot: SnapshotState, + result: BackendSnapshotResult, +): SnapshotState { + const merged = { ...snapshot }; + if (result.truncated === true || merged.truncated === true) merged.truncated = true; + else if (result.truncated !== undefined) merged.truncated = result.truncated; + if (result.quality && merged.snapshotQuality === undefined) { + merged.snapshotQuality = result.quality; + } + return merged; +} + export async function readText( runtime: AgentDeviceRuntime, capture: CapturedSnapshot, diff --git a/src/commands/interaction/runtime/selector-read.test.ts b/src/commands/interaction/runtime/selector-read.test.ts index 22e343e639..72d73a160a 100644 --- a/src/commands/interaction/runtime/selector-read.test.ts +++ b/src/commands/interaction/runtime/selector-read.test.ts @@ -264,6 +264,262 @@ test('runtime is validates selector predicates', async () => { }); }); +test('runtime is absent passes when the selector has no match', async () => { + const device = createSelectorDevice( + makeSnapshotState([{ index: 0, depth: 0, type: 'StaticText', label: 'Current screen' }]), + ); + + const result = await device.selectors.is({ + session: 'default', + predicate: 'absent', + selector: 'label="Removed row"', + }); + + assert.equal(result.predicate, 'absent'); + assert.equal(result.pass, true); +}); + +test('runtime is absent uses one readAny capture without requesting rects', async () => { + const snapshot = makeSnapshotState([ + { index: 0, depth: 0, type: 'StaticText', label: 'Current screen' }, + ]); + let captures = 0; + let captureOptions: BackendSnapshotOptions | undefined; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async (_context, options) => { + captures += 1; + captureOptions = options; + return { snapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + }); + + const result = await device.selectors.is({ + session: 'default', + predicate: 'absent', + selector: 'label="Removed row"', + }); + + assert.deepEqual(result, { + predicate: 'absent', + pass: true, + selector: 'label="Removed row"', + matches: 0, + }); + assert.equal(captures, 1); + assert.equal(captureOptions?.includeRects, false); + assert.equal(captureOptions?.depth, undefined); + assert.equal(captureOptions?.scope, undefined); +}); + +test('runtime is absent reports one matching node without visibility or geometry claims', async () => { + const snapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeButton', + identifier: 'save', + label: 'Save', + visibleToUser: false, + }, + ]); + const device = createSelectorDevice(snapshot); + + await assert.rejects( + device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Save"' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.deepEqual(error.details, { + command: 'is', + reason: 'predicate_failed', + predicate: 'absent', + selector: 'label="Save"', + matches: 1, + observation: 'present', + firstMatch: { id: 'save', role: 'button', label: 'Save', text: 'Save' }, + }); + assert.equal(/visible|hidden|rect/i.test(error.message), false); + return true; + }, + ); +}); + +test('runtime is absent reports the match count and refinement hint for multiple matches', async () => { + const snapshot = makeSnapshotState([ + { index: 0, depth: 0, type: 'Button', identifier: 'save-one', label: 'Save' }, + { index: 1, depth: 0, type: 'Button', identifier: 'save-two', label: 'Save' }, + ]); + const device = createSelectorDevice(snapshot); + + await assert.rejects( + device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Save"' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'predicate_failed'); + assert.equal(error.details?.matches, 2); + assert.deepEqual(error.details?.firstMatch, { + id: 'save-one', + role: 'button', + label: 'Save', + text: 'Save', + }); + assert.equal(error.details?.hint, 'Refine the selector to match no elements.'); + return true; + }, + ); +}); + +test('runtime is absent keeps the first matching selector alternative as the readAny domain', async () => { + const snapshot = makeSnapshotState([ + { index: 0, depth: 0, type: 'Button', identifier: 'save-one', label: 'Save' }, + { index: 1, depth: 0, type: 'Button', identifier: 'save-two', label: 'Save' }, + { index: 2, depth: 0, type: 'Button', identifier: 'gone', label: 'Gone' }, + ]); + const device = createSelectorDevice(snapshot); + + await assert.rejects( + device.selectors.is({ + session: 'default', + predicate: 'absent', + selector: 'label="Save" || label="Gone"', + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'predicate_failed'); + assert.equal(error.details?.observation, 'present'); + assert.equal(error.details?.matches, 2); + assert.deepEqual(error.details?.firstMatch, { + id: 'save-one', + role: 'button', + label: 'Save', + text: 'Save', + }); + return true; + }, + ); +}); + +test('runtime is absent fails closed for a sparse capture and preserves the last session snapshot', async () => { + const initial = makeSnapshotState([{ index: 0, depth: 0, type: 'StaticText', label: 'Initial' }]); + const sparse = makeSnapshotState([], { + snapshotQuality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, + }); + const sessions = createMemorySessionStore([{ name: 'default', snapshot: initial }]); + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => ({ snapshot: sparse }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions, + policy: localCommandPolicy(), + }); + + await assert.rejects( + device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Gone"' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'predicate_failed'); + assert.equal(error.details?.observation, 'sparse'); + assert.equal(error.details?.matches, 0); + assert.deepEqual(error.details?.snapshotQuality, sparse.snapshotQuality); + return true; + }, + ); + assert.equal((await sessions.get('default'))?.snapshot?.nodes[0]?.label, 'Initial'); +}); + +test('runtime is absent fails closed for a truncated capture even with zero matches', async () => { + const snapshot = makeSnapshotState([]); + const device = createSelectorDevice(snapshot, { + captureSnapshot: async () => ({ snapshot, truncated: true }), + }); + + await assert.rejects( + device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Gone"' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'predicate_failed'); + assert.equal(error.details?.observation, 'truncated'); + assert.equal(error.details?.matches, 0); + assert.equal(error.details?.truncated, true); + return true; + }, + ); +}); + +test('runtime is absent maps an unreadable capture to typed predicate failure evidence', async () => { + const device = createSelectorDevice(makeSnapshotState([]), { + captureSnapshot: async () => { + throw new AppError('COMMAND_FAILED', 'accessibility capture failed', { + reason: 'capture-failed', + }); + }, + }); + + await assert.rejects( + device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Gone"' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'predicate_failed'); + assert.equal(error.details?.observation, 'unreadable'); + assert.equal(error.details?.captureErrorCode, 'COMMAND_FAILED'); + assert.equal(error.details?.captureErrorReason, 'capture-failed'); + assert.equal(error.details?.matches, 0); + return true; + }, + ); +}); + +test('runtime is absent refuses depth and scope before capture with typed invalid arguments', async () => { + let captures = 0; + const device = createSelectorDevice(makeSnapshotState([]), { + captureSnapshot: async () => { + captures += 1; + return { snapshot: makeSnapshotState([]) }; + }, + }); + + for (const [option, value] of [ + ['scope', 'Login'], + ['depth', 2], + ] as const) { + await assert.rejects( + device.selectors.is({ + session: 'default', + predicate: 'absent', + selector: 'label="Gone"', + [option]: value, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal(error.details?.command, 'is'); + assert.equal(error.details?.predicate, 'absent'); + assert.equal(error.details?.rejectedOption, option); + return true; + }, + ); + } + assert.equal(captures, 0); +}); + test('runtime find get_text reads the matched node', async () => { const device = createSelectorDevice(selectorReadSnapshot(), { readText: 'Continue', diff --git a/src/commands/interaction/runtime/selector-read.ts b/src/commands/interaction/runtime/selector-read.ts index ebd79dc34a..1f697da574 100644 --- a/src/commands/interaction/runtime/selector-read.ts +++ b/src/commands/interaction/runtime/selector-read.ts @@ -11,6 +11,7 @@ import { parseFindSelectorExpression, type FindAction, type FindLocator, + type IsPredicate, } from '@agent-device/selectors'; import { listSelectorPipelineMatches, @@ -21,7 +22,7 @@ import { SELECTOR_PIPELINE_POLICIES } from '../../../core/selector-pipeline-poli import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; -import { AppError } from '@agent-device/kernel/errors'; +import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; import type { ElementTarget, FindReadResult, @@ -41,6 +42,12 @@ import { } from './selector-read-shared.ts'; import { findSnapshotScope, sparseSelectorSnapshotError } from './selector-read-utils.ts'; import { deriveSelectorCapturePolicy } from './selector-capture-policy.ts'; +import { absenceCaptureOptionRefusal } from '../../../core/absence-observation.ts'; +import { + absenceCaptureOptionError, + absenceUnreadableError, +} from '../../../core/absence-observation-errors.ts'; +import { resolveAbsenceObservation } from '../../../core/absence-observation-resolution.ts'; import { createWaitPolling, type WaitPollDeadline, waitTimeoutError } from './wait-polling.ts'; import { createSelectorWaitCommands, @@ -111,7 +118,7 @@ export type GetAttrsCommandOptions = CommandContext & export type IsCommandOptions = CommandContext & SelectorSnapshotOptions & { - predicate: 'visible' | 'hidden' | 'exists' | 'editable' | 'selected' | 'focused' | 'text'; + predicate: IsPredicate; selector: string; expectedText?: string; /** ADR 0012 step 4: replay-only post-resolution guard; see resolution.ts. */ @@ -125,7 +132,7 @@ export type IsCommandResult = { matches?: number; text?: string; selectorChain?: string[]; - /** ADR 0012 decision 3 / #1349: the resolved node and its tree, for record-time evidence (absent for `exists`). */ + /** ADR 0012 decision 3 / #1349: the resolved node and its tree, for record-time evidence (absent for presence-only predicates). */ node?: SnapshotNode; preActionNodes?: SnapshotNode[]; }; @@ -285,52 +292,83 @@ export const isCommand: RuntimeCommand = asyn ): Promise => { const admitted = checkIsPredicate(options.predicate); if (!admitted.ok) throw new AppError(admitted.code, admitted.message, { hint: admitted.hint }); - // Admission normalizes case, so every decision below reads the ADMITTED value: the raw - // option would send an uppercase predicate past the gate and then evaluate it against - // lower-case branches, admitting `EXISTS`/`TEXT` and returning the wrong answer. const predicate = admitted.predicate; + if (predicate === 'absent') { + const refusedOption = absenceCaptureOptionRefusal(options); + if (refusedOption) { + throw absenceCaptureOptionError(refusedOption); + } + } if (predicate === 'text' && !options.expectedText) { throw new AppError('INVALID_ARGS', IS_TEXT_VALUE_REQUIRED_MESSAGE); } const selectorExpression = options.selector; - const capture = await captureSelectorSnapshot(runtime, options, { - updateSession: true, - ...deriveSelectorCapturePolicy(predicate), - }); - - if (predicate === 'exists') { - // `readAny`, the same row find's read actions use: presence is the - // question, so any match count passes and the first one answers. The row - // already documented itself as serving `exists`, but this branch used to - // reach the engine directly — the claim was true of the docs and not of - // the code (#1630). - const matched = await resolveSelectorPipeline( - SELECTOR_PIPELINE_POLICIES.readAny, - capture.snapshot.nodes, + const capture = await captureIsSnapshot(runtime, options, predicate, selectorExpression); + if (predicate === 'exists') + return await resolveExistsPredicate(runtime, capture, selectorExpression); + if (predicate === 'absent') { + return await resolveAbsenceObservation( + capture.snapshot, selectorExpression, - { platform: runtime.backend.platform }, + runtime.backend.platform, + ); + } + return await resolveAssertedPredicate(runtime, options, capture, predicate, selectorExpression); +}; + +async function captureIsSnapshot( + runtime: AgentDeviceRuntime, + options: IsCommandOptions, + predicate: IsPredicate, + selectorExpression: string, +): Promise { + try { + return await captureSelectorSnapshot(runtime, options, { + updateSession: true, + ...deriveSelectorCapturePolicy(predicate), + }); + } catch (error) { + if (predicate !== 'absent' || isRequestCanceledError(error)) throw error; + throw absenceUnreadableError(selectorExpression, error); + } +} + +async function resolveExistsPredicate( + runtime: AgentDeviceRuntime, + capture: CapturedSnapshot, + selectorExpression: string, +): Promise { + const matched = await resolveSelectorPipeline( + SELECTOR_PIPELINE_POLICIES.readAny, + capture.snapshot.nodes, + selectorExpression, + { platform: runtime.backend.platform }, + ); + if (matched.kind !== 'target') { + throw new AppError( + 'COMMAND_FAILED', + formatSelectorFailure(selectorExpression, [], { unique: false }), + { + hint: selectorFailureHint([]), + }, ); - if (matched.kind !== 'target') { - throw new AppError( - 'COMMAND_FAILED', - formatSelectorFailure(selectorExpression, [], { unique: false }), - { - hint: selectorFailureHint([]), - }, - ); - } - return { - predicate: predicate, - pass: true, - selector: matched.selector, - matches: matched.matches, - selectorChain: readSelectorAlternatives(selectorExpression), - }; } + return { + predicate: 'exists', + pass: true, + selector: matched.selector, + matches: matched.matches, + selectorChain: readSelectorAlternatives(selectorExpression), + }; +} - // `readUnique` is the fail-closed row: an ambiguous screen reports the same - // refusal as no match at all, because `is` must never guess which duplicate - // it answered about. +async function resolveAssertedPredicate( + runtime: AgentDeviceRuntime, + options: IsCommandOptions, + capture: CapturedSnapshot, + predicate: Exclude, + selectorExpression: string, +): Promise { const outcome = await resolveSelectorPipeline( SELECTOR_PIPELINE_POLICIES.readUnique, capture.snapshot.nodes, @@ -354,10 +392,9 @@ export const isCommand: RuntimeCommand = asyn }, ); } - const resolved = outcome; const result = evaluateIsPredicate({ - predicate: predicate, - node: resolved.node, + predicate, + node: outcome.node, nodes: capture.snapshot.nodes, expectedText: options.expectedText, platform: runtime.backend.platform, @@ -365,26 +402,26 @@ export const isCommand: RuntimeCommand = asyn if (!result.pass) { throw new AppError( 'COMMAND_FAILED', - `is ${predicate} failed for selector ${resolved.selector}: ${result.details}`, + `is ${predicate} failed for selector ${outcome.selector}: ${result.details}`, { command: 'is', - reason: 'predicate_failed', + reason: INTERACTION_ERROR_REASONS.predicateFailed, predicate: predicate, - selector: resolved.selector, + selector: outcome.selector, predicateDetails: result.details, }, ); } return { - predicate: predicate, + predicate, pass: true, - selector: resolved.selector, + selector: outcome.selector, ...(predicate === 'text' ? { text: result.actualText } : {}), selectorChain: readSelectorAlternatives(selectorExpression), - node: resolved.node, + node: outcome.node, preActionNodes: capture.snapshot.nodes, }; -}; +} export const isVisibleCommand: RuntimeCommand = async ( runtime, diff --git a/src/commands/interaction/selectors.ts b/src/commands/interaction/selectors.ts index 27b0424e0e..caddfa6c69 100644 --- a/src/commands/interaction/selectors.ts +++ b/src/commands/interaction/selectors.ts @@ -8,6 +8,8 @@ import { normalizeIsPositionals, UNSUPPORTED_FIND_ACTION_HINT, } from '@agent-device/selectors'; +import { absenceCaptureOptionRefusal } from '../../core/absence-observation.ts'; +import { absenceCaptureOptionError } from '../../core/absence-observation-errors.ts'; import { direct, optionalCliNumber, @@ -139,6 +141,12 @@ function readIsOptionsFromPositionals(positionals: string[], flags: CliFlags): I // check this replaced compared the raw token, so the CLI used to be stricter than the // executor it hands the command to. const predicate = admitted.predicate; + if (predicate === 'absent') { + const refusedOption = absenceCaptureOptionRefusal(base); + if (refusedOption) { + throw absenceCaptureOptionError(refusedOption); + } + } const split = splitRequiredSelector(normalized.slice(1), { preferTrailingValue: predicate === 'text', }); diff --git a/src/core/absence-observation-errors.ts b/src/core/absence-observation-errors.ts new file mode 100644 index 0000000000..acc6b0fd58 --- /dev/null +++ b/src/core/absence-observation-errors.ts @@ -0,0 +1,76 @@ +import { asAppError, AppError } from '@agent-device/kernel/errors'; +import { INTERACTION_ERROR_REASONS } from '@agent-device/contracts/interaction-error'; +import { + absenceCaptureOptionMessage, + type AbsenceCaptureOption, + type AbsenceObservation, +} from './absence-observation.ts'; + +export function absenceCaptureOptionError(option: AbsenceCaptureOption): AppError { + return new AppError('INVALID_ARGS', absenceCaptureOptionMessage(option), { + command: 'is', + predicate: 'absent', + rejectedOption: option, + }); +} + +export function absenceObservationError( + selector: string, + observation: AbsenceObservation, +): AppError { + const firstMatch = 'firstMatch' in observation ? observation.firstMatch : undefined; + const details = { + command: 'is', + reason: INTERACTION_ERROR_REASONS.predicateFailed, + predicate: 'absent', + selector, + matches: observation.matches, + observation: observation.kind, + ...(firstMatch ? { firstMatch } : {}), + ...(observation.kind === 'sparse' ? { snapshotQuality: observation.quality } : {}), + ...(observation.kind === 'truncated' ? { truncated: true } : {}), + }; + if (observation.kind === 'present') { + const multiple = observation.matches > 1; + return new AppError( + 'COMMAND_FAILED', + `is absent failed for selector ${selector}: ${observation.matches} match${multiple ? 'es' : ''} found`, + { + ...details, + ...(multiple ? { hint: 'Refine the selector to match no elements.' } : {}), + }, + ); + } + return new AppError( + 'COMMAND_FAILED', + `is absent could not prove absence for selector ${selector}: ${ + observation.kind === 'sparse' ? 'capture was sparse' : 'capture was truncated' + }`, + { + ...details, + hint: 'Retry after the accessibility capture is complete.', + }, + ); +} + +export function absenceUnreadableError(selector: string, error: unknown): AppError { + const cause = asAppError(error); + return new AppError( + 'COMMAND_FAILED', + `is absent could not prove absence for selector ${selector}: capture was unreadable`, + { + command: 'is', + reason: INTERACTION_ERROR_REASONS.predicateFailed, + predicate: 'absent', + selector, + matches: 0, + observation: 'unreadable', + captureErrorCode: cause.code, + ...(typeof cause.details?.reason === 'string' + ? { captureErrorReason: cause.details.reason } + : {}), + hint: 'Retry after the accessibility capture is readable.', + }, + cause, + ); +} diff --git a/src/core/absence-observation-resolution.ts b/src/core/absence-observation-resolution.ts new file mode 100644 index 0000000000..f8fc1aa1c1 --- /dev/null +++ b/src/core/absence-observation-resolution.ts @@ -0,0 +1,47 @@ +import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; +import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import { resolveSelectorPipeline } from './selector-pipeline.ts'; +import { SELECTOR_PIPELINE_POLICIES } from './selector-pipeline-policy.ts'; +import { classifyAbsenceObservation } from './absence-observation.ts'; +import { absenceObservationError } from './absence-observation-errors.ts'; + +export type AbsenceObservationResult = { + predicate: 'absent'; + pass: true; + selector: string; + matches: 0; +}; + +export async function resolveAbsenceObservation( + snapshot: SnapshotState, + selectorExpression: string, + platform: Platform | PublicPlatform, +): Promise { + const matched = await resolveSelectorPipeline( + SELECTOR_PIPELINE_POLICIES.readAny, + snapshot.nodes, + selectorExpression, + { platform }, + ); + const matches = + matched.kind === 'target' || matched.kind === 'ambiguous' + ? matched.matchedNodes + : matched.kind === 'occluded' + ? [matched.node] + : []; + const observation = classifyAbsenceObservation(snapshot, matches); + if (observation.kind !== 'absent') { + throw absenceObservationError( + matched.kind === 'target' || matched.kind === 'ambiguous' + ? matched.selector + : selectorExpression, + observation, + ); + } + return { + predicate: 'absent', + pass: true, + selector: selectorExpression, + matches: observation.matches, + }; +} diff --git a/src/core/absence-observation.test.ts b/src/core/absence-observation.test.ts new file mode 100644 index 0000000000..fb55806ab5 --- /dev/null +++ b/src/core/absence-observation.test.ts @@ -0,0 +1,94 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { makeSnapshotState } from '../__tests__/test-utils/snapshot-builders.ts'; +import { + absenceCaptureOptionMessage, + absenceCaptureOptionRefusal, + classifyAbsenceObservation, +} from './absence-observation.ts'; + +test('classifies a complete capture with no selector matches as absent', () => { + const observation = classifyAbsenceObservation(makeSnapshotState([]), []); + + assert.deepEqual(observation, { kind: 'absent', matches: 0 }); +}); + +test('classifies matches as present and exposes only stable first-match fields', () => { + const first = makeSnapshotState([ + { + index: 0, + type: 'XCUIElementTypeButton', + identifier: 'save', + label: 'Save', + value: 'Save value', + rect: { x: 10, y: 20, width: 80, height: 30 }, + visibleToUser: false, + }, + ]).nodes[0]!; + const second = makeSnapshotState([{ index: 0, type: 'XCUIElementTypeButton', label: 'Other' }]) + .nodes[0]!; + + const observation = classifyAbsenceObservation(makeSnapshotState([]), [first, second]); + + assert.deepEqual(observation, { + kind: 'present', + matches: 2, + firstMatch: { id: 'save', role: 'button', label: 'Save', text: 'Save' }, + }); + assert.equal('rect' in observation.firstMatch, false); + assert.equal('visibleToUser' in observation.firstMatch, false); +}); + +test('classifies sparse and truncated captures before evaluating zero matches', () => { + const sparse = classifyAbsenceObservation( + makeSnapshotState([], { + snapshotQuality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, + }), + [], + ); + const truncated = classifyAbsenceObservation(makeSnapshotState([], { truncated: true }), []); + + assert.deepEqual(sparse, { + kind: 'sparse', + matches: 0, + quality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, + }); + assert.deepEqual(truncated, { kind: 'truncated', matches: 0 }); +}); + +test('classifies a quality-less legacy iOS root-only capture as sparse', () => { + const snapshot = makeSnapshotState([{ index: 0, type: 'XCUIElementTypeApplication' }], { + backend: 'xctest', + producer: 'apple-runner', + }); + + assert.deepEqual(classifyAbsenceObservation(snapshot, []), { + kind: 'sparse', + matches: 0, + quality: { + state: 'sparse', + backend: 'tree', + reason: 'legacy iOS capture exposed only the application root', + reasonCode: 'sparse-tree', + }, + }); +}); + +test('refuses scoped and depth-limited absence captures', () => { + assert.equal(absenceCaptureOptionRefusal({ scope: 'Login' }), 'scope'); + assert.equal(absenceCaptureOptionRefusal({ depth: 2 }), 'depth'); + assert.equal(absenceCaptureOptionRefusal({ scope: 'Login', depth: 2 }), 'scope'); + assert.equal(absenceCaptureOptionRefusal({}), undefined); + assert.match(absenceCaptureOptionMessage('scope'), /unscoped capture/); + assert.match(absenceCaptureOptionMessage('depth'), /full-depth capture/); +}); diff --git a/src/core/absence-observation.ts b/src/core/absence-observation.ts new file mode 100644 index 0000000000..14890825af --- /dev/null +++ b/src/core/absence-observation.ts @@ -0,0 +1,111 @@ +import { extractNodeText, normalizeType } from '@agent-device/contracts/snapshot'; +import { readNodeLocalIdentity } from '@agent-device/ad-script'; +import type { + SnapshotNode, + SnapshotQualityVerdict, + SnapshotState, +} from '@agent-device/kernel/snapshot'; + +export type AbsenceFirstMatch = { + id?: string; + role: string; + label?: string; + text?: string; +}; + +type SparseQuality = Pick & { + state: 'sparse'; +}; + +const LEGACY_IOS_SPARSE_QUALITY: SparseQuality = { + state: 'sparse', + backend: 'tree', + reason: 'legacy iOS capture exposed only the application root', + reasonCode: 'sparse-tree', +}; + +export type AbsenceObservation = + | { kind: 'absent'; matches: 0 } + | { kind: 'present'; matches: number; firstMatch: AbsenceFirstMatch } + | { kind: 'sparse'; matches: number; firstMatch?: AbsenceFirstMatch; quality: SparseQuality } + | { kind: 'truncated'; matches: number; firstMatch?: AbsenceFirstMatch }; + +export type AbsenceCaptureOption = 'depth' | 'scope'; + +export function absenceCaptureOptionRefusal(options: { + depth?: number; + scope?: string; +}): AbsenceCaptureOption | undefined { + if (options.scope !== undefined) return 'scope'; + if (options.depth !== undefined) return 'depth'; + return undefined; +} + +export function absenceCaptureOptionMessage(option: AbsenceCaptureOption): string { + return option === 'scope' + ? 'is absent does not support --scope; it requires an unscoped capture' + : 'is absent does not support --depth; it requires a full-depth capture'; +} + +export function classifyAbsenceObservation( + snapshot: Pick, + matches: readonly SnapshotNode[], +): AbsenceObservation { + const firstMatch = matches[0] ? stableFirstMatch(matches[0]) : undefined; + const matchCount = matches.length; + if (snapshot.truncated === true) { + return { + kind: 'truncated', + matches: matchCount, + ...(firstMatch ? { firstMatch } : {}), + }; + } + const sparseQuality = sparseQualityForSnapshot(snapshot); + if (sparseQuality) { + return { + kind: 'sparse', + matches: matchCount, + ...(firstMatch ? { firstMatch } : {}), + quality: { + state: sparseQuality.state, + backend: sparseQuality.backend, + ...(sparseQuality.reason ? { reason: sparseQuality.reason } : {}), + ...(sparseQuality.reasonCode ? { reasonCode: sparseQuality.reasonCode } : {}), + }, + }; + } + if (matchCount === 0) return { kind: 'absent', matches: 0 }; + return { kind: 'present', matches: matchCount, firstMatch: firstMatch! }; +} + +function sparseQualityForSnapshot( + snapshot: Pick, +): SparseQuality | undefined { + const quality = snapshot.snapshotQuality; + if (quality?.state === 'sparse') { + return { + state: quality.state, + backend: quality.backend, + ...(quality.reason ? { reason: quality.reason } : {}), + ...(quality.reasonCode ? { reasonCode: quality.reasonCode } : {}), + }; + } + return isLegacySparseIosInteractiveSnapshot(snapshot) ? LEGACY_IOS_SPARSE_QUALITY : undefined; +} + +export function isLegacySparseIosInteractiveSnapshot( + snapshot: Pick, +): boolean { + if (snapshot.snapshotQuality) return false; + if (snapshot.backend !== 'xctest' || snapshot.nodes.length !== 1) return false; + return normalizeType(snapshot.nodes[0]?.type ?? '') === 'application'; +} + +function stableFirstMatch(node: SnapshotNode): AbsenceFirstMatch { + const identity = readNodeLocalIdentity(node); + const text = extractNodeText(node); + return { + ...identity, + ...(text ? { text } : {}), + }; +} diff --git a/src/core/selector-pipeline-policy.ts b/src/core/selector-pipeline-policy.ts index 0cacca6be9..c9d3724235 100644 --- a/src/core/selector-pipeline-policy.ts +++ b/src/core/selector-pipeline-policy.ts @@ -145,7 +145,7 @@ export const SELECTOR_PIPELINE_POLICIES = { promotion: 'none', poll: 'none', }, - /** `is` non-exists predicates and `get attrs`. */ + /** `is` predicates other than `exists`/`absent`, and `get attrs`. */ readUnique: { resolution: SELECTOR_RESOLUTION_POLICIES.readUnique, occlusion: 'ignore', @@ -153,7 +153,7 @@ export const SELECTOR_PIPELINE_POLICIES = { promotion: 'none', poll: 'none', }, - /** `is exists` and find's read-only actions. */ + /** `is exists`/`absent` and find's read-only actions. */ readAny: { resolution: SELECTOR_RESOLUTION_POLICIES.readAny, occlusion: 'ignore', diff --git a/src/daemon/__tests__/is-runtime.test.ts b/src/daemon/__tests__/is-runtime.test.ts index 0986b71a88..9f840f4f90 100644 --- a/src/daemon/__tests__/is-runtime.test.ts +++ b/src/daemon/__tests__/is-runtime.test.ts @@ -26,7 +26,7 @@ beforeEach(() => { mockRunAppleRunnerCommand.mockResolvedValue({}); }); -// `is` answers every one of its seven predicates from the resolved capture — `isCommand` never +// `is` answers every one of its eight predicates from the resolved capture — `isCommand` never // reaches `backend.readText`. So its whole platform execution is the request-bound capture, and // these cases bind at `inspectFacts` / `bindDevice`, never at `core/dispatch-resolve.ts`. @@ -54,8 +54,12 @@ function buttonSnapshot(): SnapshotResult { }; } -function isRequest(session: string, positionals: readonly string[]): DaemonRequest { - return { token: 't', session, command: 'is', positionals: [...positionals], flags: {} }; +function isRequest( + session: string, + positionals: readonly string[], + flags: DaemonRequest['flags'] = {}, +): DaemonRequest { + return { token: 't', session, command: 'is', positionals: [...positionals], flags }; } test('an admitted is inspects once, binds once, and answers through the bound capture', async () => { @@ -77,6 +81,103 @@ test('an admitted is inspects once, binds once, and answers through the bound ca expect(fixture.captures.length).toBeGreaterThan(0); }); +test('is absent uses the bound readAny capture for selector-first input without rects', async () => { + const fixture = selectorCaptureFixture({ + snapshot: () => ({ nodes: [], backend: 'xctest', producer: 'apple-runner' }), + }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-absent', makeIosAppSession('is-absent')); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-absent', ['label="Removed row"', 'absent']), + sessionName: 'is-absent', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(true); + if (response?.ok) { + expect(response.data).toMatchObject({ + predicate: 'absent', + pass: true, + matches: 0, + }); + } + expect(fixture.captures).toHaveLength(1); + expect(fixture.captures[0]?.options?.includeRects).toBe(false); +}); + +test('is absent fails closed for a quality-less legacy iOS root-only capture', async () => { + const fixture = selectorCaptureFixture({ + snapshot: () => ({ + nodes: [{ index: 0, type: 'XCUIElementTypeApplication' }], + backend: 'xctest', + producer: 'apple-runner', + }), + }); + const sessionStore = makeSessionStore(); + sessionStore.set('is-legacy-sparse', makeIosAppSession('is-legacy-sparse')); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-legacy-sparse', ['absent', 'label="Removed row"']), + sessionName: 'is-legacy-sparse', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(false); + if (response?.ok === false) { + expect(response.error.code).toBe('COMMAND_FAILED'); + expect(response.error.details).toMatchObject({ + command: 'is', + reason: 'predicate_failed', + predicate: 'absent', + observation: 'sparse', + matches: 0, + snapshotQuality: { + state: 'sparse', + backend: 'tree', + reasonCode: 'sparse-tree', + }, + }); + } + expect(fixture.captures).toHaveLength(1); +}); + +test('is absent rejects depth and scope before binding with typed invalid arguments', async () => { + const fixture = selectorCaptureFixture(); + const sessionStore = makeSessionStore(); + sessionStore.set('is-absent-flags', makeIosAppSession('is-absent-flags')); + + for (const [flag, value] of [ + ['snapshotScope', 'Login'], + ['snapshotDepth', 2], + ] as const) { + const response = await dispatchIsViaRuntime({ + req: isRequest('is-absent-flags', ['absent', 'label="Gone"'], { [flag]: value }), + sessionName: 'is-absent-flags', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(false); + if (response?.ok === false) { + expect(response.error.code).toBe('INVALID_ARGS'); + expect(response.error.details).toMatchObject({ + command: 'is', + predicate: 'absent', + rejectedOption: flag === 'snapshotScope' ? 'scope' : 'depth', + }); + } + } + expect(fixture.inspections).toEqual([]); + expect(fixture.binds).toEqual([]); + expect(fixture.captures).toEqual([]); +}); + test('an unavailable capture fact refuses before any bind', async () => { // The watchOS sentinel shape: capability-supported today, no snapshot backend at the owner. const fixture = selectorCaptureFixture({ capture: unavailableCapture }); diff --git a/src/daemon/interaction/internal/__tests__/interaction-target-evidence.test.ts b/src/daemon/interaction/internal/__tests__/interaction-target-evidence.test.ts index 211a278732..7ea725ec41 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-target-evidence.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-target-evidence.test.ts @@ -413,6 +413,37 @@ test('is exists while recording stays intentionally unannotated (deferred covera expect(recordedAction?.targetEvidence).toBeUndefined(); }); +test('is absent while recording stays an ordinary unannotated observation', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'recording-is-absent'; + sessionStore.set(sessionName, makeSessionWithSnapshot(sessionName, { recording: true })); + mockSnapshotWithSaveButton(); + + const response = await runCommand(sessionStore, sessionName, 'is', [ + 'absent', + 'label="Removed row"', + ]); + + expect(response?.ok).toBe(true); + if (response?.ok) { + expect(response.data).toEqual({ + predicate: 'absent', + pass: true, + selector: 'label="Removed row"', + matches: 0, + }); + } + const recordedAction = sessionStore.get(sessionName)?.actions[0]; + expect(recordedAction?.command).toBe('is'); + expect(recordedAction?.targetEvidence).toBeUndefined(); + expect(recordedAction?.result).toEqual({ + predicate: 'absent', + pass: true, + selector: 'label="Removed row"', + matches: 0, + }); +}); + test('is visible without recording never computes target-v1 evidence', async () => { const sessionStore = makeSessionStore(); const sessionName = 'non-recording-is'; diff --git a/src/daemon/replay/internal/__tests__/session-replay-dispatch-selector-miss.test.ts b/src/daemon/replay/internal/__tests__/session-replay-dispatch-selector-miss.test.ts index 78559219e3..60f9eefb02 100644 --- a/src/daemon/replay/internal/__tests__/session-replay-dispatch-selector-miss.test.ts +++ b/src/daemon/replay/internal/__tests__/session-replay-dispatch-selector-miss.test.ts @@ -34,6 +34,7 @@ import path from 'node:path'; import { runReplayForTest } from '../../__tests__/replay-command-fixture.ts'; import { SessionStore } from '../../../session-store.ts'; import { AppError } from '@agent-device/kernel/errors'; +import type { DaemonRequest } from '../../../types.ts'; import { captureSnapshotWithInteractor } from '../../../snapshot-interactor-capture.ts'; import { makeIosSession } from '../../../../__tests__/test-utils/session-factories.ts'; import { @@ -190,6 +191,54 @@ test('(b) an UNANNOTATED press whose dispatch throws a selector-miss yields REPL expect(cause.code).toBe('COMMAND_FAILED'); }); +test('an authored is absent predicate failure remains an action-failure, never an identity mismatch', async () => { + const root = mkdtempForTestSync('agent-device-replay-is-absent-failure-'); + const { sessionStore, sessionName } = setupSession(root); + const filePath = writeReplayFile(root, ['is absent label="Gone"']); + + mockDispatchCommand.mockResolvedValue({ + nodes: bottomTabsRealCaptureFixture(), + truncated: false, + backend: 'xctest', + }); + + const invoked: DaemonRequest[] = []; + const response = await runReplayForTest({ + req: baseReq({ positionals: [filePath] }), + sessionName, + logPath: path.join(root, 'daemon.log'), + sessionStore, + invoke: async (req) => { + invoked.push(req); + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: 'is absent failed for selector label="Gone": 1 match found', + details: { + command: 'is', + reason: 'predicate_failed', + predicate: 'absent', + selector: 'label="Gone"', + matches: 1, + observation: 'present', + }, + }, + }; + }, + }); + + expect(invoked).toHaveLength(1); + expect(invoked[0]?.command).toBe('is'); + expect(invoked[0]?.positionals).toEqual(['absent', 'label="Gone"']); + const { divergence, targetBinding } = assertDivergenceShape(response); + expect(divergence.kind).toBe('action-failure'); + expect(targetBinding).toBeUndefined(); + const cause = divergence.cause as { code: string; message: string }; + expect(cause.code).toBe('COMMAND_FAILED'); + expect(cause.message).toContain('is absent failed'); +}); + test('(c) a fill selector-miss thrown at dispatch yields REPLAY_DIVERGENCE, not COMMAND_FAILED', async () => { const root = mkdtempForTestSync('agent-device-replay-dispatch-miss-fill-'); const { sessionStore, sessionName } = setupSession(root); diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index 11d9dae50d..28dca84a5c 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -1,5 +1,4 @@ import type { CommandFlags } from '@agent-device/contracts/command'; -import { normalizeType } from '@agent-device/contracts/snapshot'; import type { BackendSnapshotResult } from '../backend.ts'; import { buildSnapshotPresentationKey, @@ -15,6 +14,7 @@ import { getActiveAndroidSnapshotFreshness } from './session-snapshot-freshness. import { isPostGestureStabilizationPending } from './deferred-interaction-outcome.ts'; import type { BoundSelectorCapture } from './selector-capture-binding.ts'; import { buildRuntimeCaptureInput } from './snapshot-runtime-capture-input.ts'; +import { isLegacySparseIosInteractiveSnapshot } from '../core/absence-observation.ts'; const SELECTOR_CAPTURE_CACHE_TTL_MS = 750; @@ -315,9 +315,3 @@ function updateSessionSnapshot(params: { setSessionSnapshot(session, snapshot); sessionStore.set(sessionName, session); } - -function isLegacySparseIosInteractiveSnapshot(snapshot: SnapshotState): boolean { - if (snapshot.snapshotQuality) return false; - if (snapshot.backend !== 'xctest' || snapshot.nodes.length !== 1) return false; - return normalizeType(snapshot.nodes[0]?.type ?? '') === 'application'; -} diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index 274aebb908..f90ddf4184 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -46,6 +46,8 @@ import { type BoundSelectorOperations, } from './selector-capture-binding.ts'; import { dispatchConditionalWaitSelector } from './wait-conditional-selector.ts'; +import { absenceCaptureOptionRefusal } from '../core/absence-observation.ts'; +import { absenceCaptureOptionError } from '../core/absence-observation-errors.ts'; export async function dispatchFindReadOnlyViaRuntime( params: SelectorRuntimeParams, @@ -205,6 +207,16 @@ export async function dispatchIsViaRuntime( ); } const { predicate, selectorExpression, expectedText } = checked; + if (predicate === 'absent') { + const refusedOption = absenceCaptureOptionRefusal({ + depth: req.flags?.snapshotDepth, + scope: req.flags?.snapshotScope, + }); + if (refusedOption) { + const error = absenceCaptureOptionError(refusedOption); + return errorResponse(error.code, error.message, error.details); + } + } // ADR 0012 decision 3 / #1349: a guarded replay dispatch resolves through the snapshot path so // the post-resolution identity guard runs against the resolution tree. const replayTargetGuard = req.internal?.replayTargetGuard; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 43f7e7293c..eca4b8d87f 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -174,7 +174,7 @@ agent-device close --platform web - `web doctor` verifies the managed backend after setup. - The managed install respects `--state-dir` and `AGENT_DEVICE_STATE_DIR`. - Web automation requires Node 24+. -- Supported through `agent-device`: URL open, snapshot refs, `get text/attrs`, `is visible/exists/text`, `find text/selector`, click/press, hover, fill/type, wait, `network dump`, `audio probe`, screenshot, close, and replay scripts composed from those commands. +- Supported through `agent-device`: URL open, snapshot refs, `get text/attrs`, `is visible/hidden/exists/absent/focused/text`, `find text/selector`, click/press, hover, fill/type, wait, `network dump`, `audio probe`, screenshot, close, and replay scripts composed from those commands. - `hover <@ref|selector|x y>` moves the pointer without pressing so hover-gated UI (row toolbars, menus) appears. Add `--settle` to read what it revealed instead of taking another snapshot. `hover @ref` hovers the browser's own element handle; like `click @ref --settle`, the `--settle` diff needs a selector or coordinate target on web because web refs carry no geometry. - `audio probe start [durationSeconds] [bucketMs]` samples HTML media elements into compact RMS/peak dBFS buckets while the page keeps running. The first timing positional is seconds; the second is milliseconds. - URL-backed web media may be routed through the probe `AudioContext` while observed. Use `audio probe status` to poll partial buckets and `audio probe stop` to end the probe early. @@ -514,6 +514,7 @@ Actions: `click` (default; `press`/`tap` are aliases), `list`, `focus`, `fill`, ```bash agent-device is visible 'role="button" label="Continue"' agent-device is exists 'id="primary-cta"' +agent-device is absent 'label="Loading..."' agent-device is hidden 'text="Loading..."' agent-device is editable 'id="email"' agent-device is selected 'label="Wi-Fi"' @@ -521,13 +522,14 @@ agent-device is text 'id="greeting"' "Welcome back" ``` - `is` evaluates UI predicates against a selector expression and exits non-zero on failure. -- Supported predicates are `visible`, `hidden`, `exists`, `editable`, `selected`, and `text`. +- Supported predicates are `visible`, `hidden`, `exists`, `absent`, `editable`, `selected`, `focused`, and `text`. - `is visible` checks whether the resolved element is present in the current visible snapshot viewport. A node without its own rect still passes when a visible ancestor within the viewport provides the on-screen geometry. - `is exists` only checks whether the selector matches in the current snapshot. +- `is absent` passes only when the selector has zero matches in one readable, complete, unscoped, full-depth accessibility capture. It does not mean hidden; `--scope` and `--depth` are rejected, and sparse, unreadable, or truncated captures fail closed. - `wait text` is a text-presence wait, not a hittability assertion. - `is text ` compares the resolved element text against the expected value. - `is` does not accept snapshot refs like `@e3`; use a selector expression instead. -- `is` accepts the same selector-oriented snapshot flags as `click`, `fill`, `get`, and `wait`. +- `is` accepts the same selector-oriented snapshot flags as `click`, `fill`, `get`, and `wait`; `is absent` rejects `--scope` and `--depth` because its proof must cover the complete unscoped tree. ## Replay From 97c555c3876e1ee4f2041302a820113db0c6cf43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 2 Sep 2026 21:50:01 +0200 Subject: [PATCH 2/2] fix: address absence assertion review feedback --- packages/ad-script/src/index.ts | 1 + scripts/__tests__/eager-closure-budgets.ts | 11 +- .../is-argument-surface-parity.test.ts | 2 +- src/commands/interaction/runtime/index.ts | 17 +- .../interaction/runtime/selector-is.test.ts | 465 ++++++++++++++++++ .../interaction/runtime/selector-is.ts | 213 ++++++++ .../interaction/runtime/selector-read.test.ts | 455 ----------------- .../interaction/runtime/selector-read.ts | 200 +------- src/commands/interaction/selectors.ts | 25 +- src/core/absence-observation.test.ts | 16 +- src/core/absence-observation.ts | 10 +- src/daemon/__tests__/is-runtime.test.ts | 44 ++ src/daemon/selector-runtime-backend.ts | 8 + .../android-emulator-e2e/coverage-manifest.ts | 5 +- .../live-automation-scenario.ts | 7 + .../ios-simulator-e2e/coverage-manifest.ts | 5 +- .../live-automation-scenario.ts | 7 + 17 files changed, 816 insertions(+), 675 deletions(-) create mode 100644 src/commands/interaction/runtime/selector-is.test.ts create mode 100644 src/commands/interaction/runtime/selector-is.ts diff --git a/packages/ad-script/src/index.ts b/packages/ad-script/src/index.ts index 6c8e11d748..2ed8efd035 100644 --- a/packages/ad-script/src/index.ts +++ b/packages/ad-script/src/index.ts @@ -21,6 +21,7 @@ export { export { parseTargetAnnotationV1Payload, serializeTargetAnnotationV1, + truncateToUtf8Bytes, utf8ByteLength, TARGET_ANNOTATION_MAX_ANCESTRY, TARGET_ANNOTATION_MAX_FIELD_BYTES, diff --git a/scripts/__tests__/eager-closure-budgets.ts b/scripts/__tests__/eager-closure-budgets.ts index 24f2045bda..c7bd3650bd 100644 --- a/scripts/__tests__/eager-closure-budgets.ts +++ b/scripts/__tests__/eager-closure-budgets.ts @@ -106,7 +106,7 @@ export function discoverFacadeEntryFiles(repoRoot: string): string[] { */ export const FACADE_BUDGETS: Readonly> = Object.freeze({ // --- @agent-device/ad-replay --- - 'packages/ad-replay/src/index.ts': 61, + 'packages/ad-replay/src/index.ts': 62, // --- @agent-device/ad-script --- 'packages/ad-script/src/index.ts': 41, @@ -233,6 +233,7 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ 'packages/contracts/src/interactor-types.ts': 1, // #2190's iOS snapshot vocabulary has type-only imports and remains a one-module entry. 'packages/contracts/src/ios-snapshot.ts': 1, + 'packages/contracts/src/is-predicate.ts': 1, 'packages/contracts/src/keyboard.ts': 1, 'packages/contracts/src/logs-runtime-plan.ts': 5, 'packages/contracts/src/managed-web-backend.ts': 1, @@ -300,7 +301,7 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ 'packages/kernel/src/snapshot.ts': 1, // --- @agent-device/maestro --- - 'packages/maestro/src/index.ts': 110, + 'packages/maestro/src/index.ts': 111, // --- @agent-device/platform-*: ADR-0019's metadata-eager/implementation-lazy façades. Each // evaluates only itself; every implementation sits behind a function-scoped `await import`. @@ -351,7 +352,7 @@ export const FACADE_BUDGETS: Readonly> = Object.freeze({ // --- @agent-device/selectors --- 'packages/selectors/src/ast.ts': 16, 'packages/selectors/src/engine.ts': 19, - 'packages/selectors/src/index.ts': 54, + 'packages/selectors/src/index.ts': 55, // --- @agent-device/xml --- 'packages/xml/src/index.ts': 3, @@ -410,9 +411,9 @@ export const HUB_BUDGETS: Readonly> = Object.freeze({ // #2148 moves output-only CLI dependencies behind call-time imports and reduces the entry // closure by two modules. // #2146 splits one eagerly reached URL utility into its client and Metro owners. - 'src/cli.ts': 379, + 'src/cli.ts': 380, 'src/platform-runtime.ts': 47, - 'src/core/command-descriptor/registry.ts': 71, + 'src/core/command-descriptor/registry.ts': 72, 'src/core/command-descriptor/platform-execution-entry.ts': 3, 'src/core/interactors/register-builtins.ts': 6, // R64 removes the perf plugin facet and keeps collector binding behind the selected runtime diff --git a/src/__tests__/is-argument-surface-parity.test.ts b/src/__tests__/is-argument-surface-parity.test.ts index f924638a17..046c3ab265 100644 --- a/src/__tests__/is-argument-surface-parity.test.ts +++ b/src/__tests__/is-argument-surface-parity.test.ts @@ -18,7 +18,7 @@ import type { CliFlags } from '@agent-device/contracts/command'; // Three surfaces validate the same rule: the daemon handler (via `checkIsArgs`), the CLI // grammar (`readInputFromCli`), and the runtime command that receives already-parsed // options (`isCommand`, covered on its own production route in -// commands/interaction/runtime/selector-read.test.ts). They used to state the rule three +// commands/interaction/runtime/selector-is.test.ts). They used to state the rule three // times, and all three had drifted: one inlined its own predicate list, one compared the raw // token so it rejected input the daemon accepts, and one dropped the usage hint. // diff --git a/src/commands/interaction/runtime/index.ts b/src/commands/interaction/runtime/index.ts index 2c304fc159..dd50012a28 100644 --- a/src/commands/interaction/runtime/index.ts +++ b/src/commands/interaction/runtime/index.ts @@ -28,9 +28,6 @@ import { getAttrsCommand, getCommand, getTextCommand, - isCommand, - isHiddenCommand, - isVisibleCommand, waitCommand, waitForTextCommand, type ElementTarget, @@ -40,21 +37,25 @@ import { type GetCommandOptions, type GetCommandResult, type GetTextCommandOptions, - type IsCommandOptions, - type IsCommandResult, - type IsSelectorCommandOptions, - type SelectorTarget, type WaitCommandOptions, type WaitCommandResult, type WaitForTextCommandOptions, } from './selector-read.ts'; +import { + isCommand, + isHiddenCommand, + isVisibleCommand, + type IsCommandOptions, + type IsCommandResult, + type IsSelectorCommandOptions, +} from './selector-is.ts'; import { gestureCommand, type GestureCommandOptions, type GestureCommandResult, } from './gesture-command.ts'; import { settleObservationCommand, type SettleObservationCommandOptions } from './settle.ts'; -import type { SettleObservation } from '@agent-device/contracts/interaction'; +import type { SelectorTarget, SettleObservation } from '@agent-device/contracts/interaction'; export type SelectorCommands = { find: RuntimeCommand; diff --git a/src/commands/interaction/runtime/selector-is.test.ts b/src/commands/interaction/runtime/selector-is.test.ts new file mode 100644 index 0000000000..c4a70c5b1b --- /dev/null +++ b/src/commands/interaction/runtime/selector-is.test.ts @@ -0,0 +1,465 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { AgentDeviceBackend, BackendSnapshotOptions } from '../../../backend.ts'; +import { createLocalArtifactAdapter } from '../../../io.ts'; +import { + createAgentDevice, + createMemorySessionStore, + localCommandPolicy, +} from '../../../runtime.ts'; +import { selector } from './selector-read-utils.ts'; +import { makeSnapshotState } from '../../../__tests__/test-utils/snapshot-builders.ts'; +import { createSelectorDevice, selectorReadSnapshot } from './__tests__/test-utils/index.ts'; +import { AppError } from '@agent-device/kernel/errors'; + +test('runtime selectors forward public snapshot options to backend capture', async () => { + const snapshot = selectorReadSnapshot(); + let captureOptions: BackendSnapshotOptions | undefined; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async (_context, options) => { + captureOptions = options; + return { snapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + }); + + await device.selectors.is({ + session: 'default', + predicate: 'exists', + selector: 'label=Continue', + depth: 2, + scope: 'Login', + raw: true, + }); + + assert.deepEqual(captureOptions, { + interactiveOnly: false, + depth: 2, + scope: 'Login', + raw: true, + includeRects: false, + }); +}); + +test('runtime visibility predicates request snapshot rects', async () => { + const snapshot = selectorReadSnapshot(); + let captureOptions: BackendSnapshotOptions | undefined; + const device = createAgentDevice({ + backend: { + platform: 'web', + captureSnapshot: async (_context, options) => { + captureOptions = options; + return { snapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + }); + + await device.selectors.isVisible(selector('label=Continue'), { + session: 'default', + }); + + assert.equal(captureOptions?.includeRects, true); +}); + +test('runtime focused predicate requests a full snapshot', async () => { + const snapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Cell', + label: 'Profiles and Accounts', + focused: true, + }, + ]); + let captureOptions: BackendSnapshotOptions | undefined; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async (_context, options) => { + captureOptions = options; + return { snapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + }); + + const result = await device.selectors.is({ + session: 'default', + predicate: 'focused', + selector: 'role=cell label="Profiles and Accounts"', + }); + + assert.equal(result.pass, true); + assert.equal(captureOptions?.interactiveOnly, false); +}); + +test('runtime focused predicate reads focused Android TV nodes from the full tree', async () => { + const fullSnapshot = makeSnapshotState( + [ + { + index: 0, + depth: 0, + type: 'TextView', + label: 'Featured', + focused: true, + hittable: false, + }, + ], + { backend: 'android' }, + ); + const interactiveSnapshot = makeSnapshotState([], { backend: 'android' }); + let captureOptions: BackendSnapshotOptions | undefined; + const device = createAgentDevice({ + backend: { + platform: 'android', + captureSnapshot: async (_context, options) => { + captureOptions = options; + return { snapshot: options?.interactiveOnly ? interactiveSnapshot : fullSnapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot: interactiveSnapshot }]), + policy: localCommandPolicy(), + }); + + const result = await device.selectors.is({ + session: 'default', + predicate: 'focused', + selector: 'label=Featured', + }); + + assert.equal(result.pass, true); + assert.equal(captureOptions?.interactiveOnly, false); +}); + +test('runtime is validates selector predicates', async () => { + const device = createSelectorDevice(selectorReadSnapshot()); + + const result = await device.selectors.is({ + session: 'default', + predicate: 'exists', + selector: 'label=Continue', + }); + + assert.deepEqual(result, { + predicate: 'exists', + pass: true, + selector: 'label=Continue', + matches: 1, + selectorChain: ['label=Continue'], + }); +}); + +test('runtime is absent passes when the selector has no match', async () => { + const device = createSelectorDevice( + makeSnapshotState([{ index: 0, depth: 0, type: 'StaticText', label: 'Current screen' }]), + ); + + const result = await device.selectors.is({ + session: 'default', + predicate: 'absent', + selector: 'label="Removed row"', + }); + + assert.equal(result.predicate, 'absent'); + assert.equal(result.pass, true); +}); + +test('runtime is absent uses one readAny capture without requesting rects', async () => { + const snapshot = makeSnapshotState([ + { index: 0, depth: 0, type: 'StaticText', label: 'Current screen' }, + ]); + let captures = 0; + let captureOptions: BackendSnapshotOptions | undefined; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async (_context, options) => { + captures += 1; + captureOptions = options; + return { snapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + }); + + const result = await device.selectors.is({ + session: 'default', + predicate: 'absent', + selector: 'label="Removed row"', + }); + + assert.deepEqual(result, { + predicate: 'absent', + pass: true, + selector: 'label="Removed row"', + matches: 0, + }); + assert.equal(captures, 1); + assert.equal(captureOptions?.includeRects, false); + assert.equal(captureOptions?.depth, undefined); + assert.equal(captureOptions?.scope, undefined); +}); + +test('runtime is absent reports one matching node without visibility or geometry claims', async () => { + const snapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeButton', + identifier: 'save', + label: 'Save', + visibleToUser: false, + }, + ]); + const device = createSelectorDevice(snapshot); + + await assert.rejects( + device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Save"' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.deepEqual(error.details, { + command: 'is', + reason: 'predicate_failed', + predicate: 'absent', + selector: 'label="Save"', + matches: 1, + observation: 'present', + firstMatch: { id: 'save', role: 'button', label: 'Save' }, + }); + assert.equal(/visible|hidden|rect/i.test(error.message), false); + return true; + }, + ); +}); + +test('runtime is absent reports the match count and refinement hint for multiple matches', async () => { + const snapshot = makeSnapshotState([ + { index: 0, depth: 0, type: 'Button', identifier: 'save-one', label: 'Save' }, + { index: 1, depth: 0, type: 'Button', identifier: 'save-two', label: 'Save' }, + ]); + const device = createSelectorDevice(snapshot); + + await assert.rejects( + device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Save"' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'predicate_failed'); + assert.equal(error.details?.matches, 2); + assert.deepEqual(error.details?.firstMatch, { + id: 'save-one', + role: 'button', + label: 'Save', + }); + assert.equal(error.details?.hint, 'Refine the selector to match no elements.'); + return true; + }, + ); +}); + +test('runtime is absent keeps the first matching selector alternative as the readAny domain', async () => { + const snapshot = makeSnapshotState([ + { index: 0, depth: 0, type: 'Button', identifier: 'save-one', label: 'Save' }, + { index: 1, depth: 0, type: 'Button', identifier: 'save-two', label: 'Save' }, + { index: 2, depth: 0, type: 'Button', identifier: 'gone', label: 'Gone' }, + ]); + const device = createSelectorDevice(snapshot); + + await assert.rejects( + device.selectors.is({ + session: 'default', + predicate: 'absent', + selector: 'label="Save" || label="Gone"', + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'predicate_failed'); + assert.equal(error.details?.observation, 'present'); + assert.equal(error.details?.matches, 2); + assert.deepEqual(error.details?.firstMatch, { + id: 'save-one', + role: 'button', + label: 'Save', + }); + return true; + }, + ); +}); + +test('runtime is absent fails closed for a sparse capture and preserves the last session snapshot', async () => { + const initial = makeSnapshotState([{ index: 0, depth: 0, type: 'StaticText', label: 'Initial' }]); + const sparse = makeSnapshotState([], { + snapshotQuality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, + }); + const sessions = createMemorySessionStore([{ name: 'default', snapshot: initial }]); + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => ({ snapshot: sparse }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions, + policy: localCommandPolicy(), + }); + + await assert.rejects( + device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Gone"' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'predicate_failed'); + assert.equal(error.details?.observation, 'sparse'); + assert.equal(error.details?.matches, 0); + assert.deepEqual(error.details?.snapshotQuality, sparse.snapshotQuality); + return true; + }, + ); + assert.equal((await sessions.get('default'))?.snapshot?.nodes[0]?.label, 'Initial'); +}); + +test('runtime is absent fails closed for a truncated capture even with zero matches', async () => { + const snapshot = makeSnapshotState([]); + const device = createSelectorDevice(snapshot, { + captureSnapshot: async () => ({ snapshot, truncated: true }), + }); + + await assert.rejects( + device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Gone"' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'predicate_failed'); + assert.equal(error.details?.observation, 'truncated'); + assert.equal(error.details?.matches, 0); + assert.equal(error.details?.truncated, true); + return true; + }, + ); +}); + +test('runtime is absent maps an unreadable capture to typed predicate failure evidence', async () => { + const device = createSelectorDevice(makeSnapshotState([]), { + captureSnapshot: async () => { + throw new AppError('COMMAND_FAILED', 'accessibility capture failed', { + reason: 'capture-failed', + }); + }, + }); + + await assert.rejects( + device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Gone"' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.reason, 'predicate_failed'); + assert.equal(error.details?.observation, 'unreadable'); + assert.equal(error.details?.captureErrorCode, 'COMMAND_FAILED'); + assert.equal(error.details?.captureErrorReason, 'capture-failed'); + assert.equal(error.details?.matches, 0); + return true; + }, + ); +}); + +test('runtime is absent refuses depth and scope before capture with typed invalid arguments', async () => { + let captures = 0; + const device = createSelectorDevice(makeSnapshotState([]), { + captureSnapshot: async () => { + captures += 1; + return { snapshot: makeSnapshotState([]) }; + }, + }); + + for (const [option, value] of [ + ['scope', 'Login'], + ['depth', 2], + ] as const) { + await assert.rejects( + device.selectors.is({ + session: 'default', + predicate: 'absent', + selector: 'label="Gone"', + [option]: value, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal(error.details?.command, 'is'); + assert.equal(error.details?.predicate, 'absent'); + assert.equal(error.details?.rejectedOption, option); + return true; + }, + ); + } + assert.equal(captures, 0); +}); + +// Regression: admission normalizes a predicate's case, and every branch below it has to read +// the ADMITTED value. Reading `options.predicate` instead let an uppercase predicate past the +// gate and then evaluated it against lower-case branches — `EXISTS` skipped its own branch and +// `TEXT` compared nothing — so the command answered wrongly instead of refusing or working. +test('runtime is admits an upper-case predicate and evaluates it as the normalized one', async () => { + const snapshot = makeSnapshotState([ + { index: 0, depth: 0, type: 'StaticText', label: 'Greeting' }, + ]); + const device = createSelectorDevice(snapshot); + + const exists = await device.selectors.is({ + session: 'default', + predicate: 'EXISTS' as 'exists', + selector: 'label=Greeting', + }); + assert.equal(exists.predicate, 'exists'); + assert.equal(exists.pass, true); + + const text = await device.selectors.is({ + session: 'default', + predicate: 'TEXT' as 'text', + selector: 'label=Greeting', + expectedText: 'Greeting', + }); + assert.equal(text.predicate, 'text'); + assert.equal(text.pass, true); + assert.equal(text.text, 'Greeting'); +}); + +test('runtime is still refuses a predicate that is not in the vocabulary', async () => { + const device = createSelectorDevice( + makeSnapshotState([{ index: 0, depth: 0, type: 'StaticText', label: 'Greeting' }]), + ); + + await assert.rejects( + async () => + await device.selectors.is({ + session: 'default', + predicate: 'shiny' as 'exists', + selector: 'label=Greeting', + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.match(String(error.details?.hint ?? ''), /is /); + return true; + }, + ); +}); diff --git a/src/commands/interaction/runtime/selector-is.ts b/src/commands/interaction/runtime/selector-is.ts new file mode 100644 index 0000000000..f9e6c5ba3b --- /dev/null +++ b/src/commands/interaction/runtime/selector-is.ts @@ -0,0 +1,213 @@ +import { + checkIsPredicate, + evaluateIsPredicate, + formatSelectorFailure, + IS_TEXT_VALUE_REQUIRED_MESSAGE, + readSelectorAlternatives, + selectorFailureHint, + type IsPredicate, +} from '@agent-device/selectors'; +import { resolveSelectorPipeline } from '../../../core/selector-pipeline.ts'; +import { SELECTOR_PIPELINE_POLICIES } from '../../../core/selector-pipeline-policy.ts'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; +import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; +import type { SelectorTarget } from '@agent-device/contracts/interaction'; +import { INTERACTION_ERROR_REASONS } from '@agent-device/contracts/interaction-error'; +import type { RuntimeCommand } from '../../runtime-types.ts'; +import { assertExpectedResolvedTarget, type ExpectedResolvedTarget } from './resolution.ts'; +import { + type CapturedSnapshot, + type SelectorSnapshotOptions, + captureSelectorSnapshot, +} from './selector-read-shared.ts'; +import { deriveSelectorCapturePolicy } from './selector-capture-policy.ts'; +import { absenceCaptureOptionRefusal } from '../../../core/absence-observation.ts'; +import { + absenceCaptureOptionError, + absenceUnreadableError, +} from '../../../core/absence-observation-errors.ts'; +import { resolveAbsenceObservation } from '../../../core/absence-observation-resolution.ts'; + +export type IsCommandOptions = CommandContext & + SelectorSnapshotOptions & { + predicate: IsPredicate; + selector: string; + expectedText?: string; + /** ADR 0012 step 4: replay-only post-resolution guard; see resolution.ts. */ + expectedResolvedTarget?: ExpectedResolvedTarget; + }; + +export type IsCommandResult = { + predicate: IsCommandOptions['predicate']; + pass: true; + selector: string; + matches?: number; + text?: string; + selectorChain?: string[]; + /** ADR 0012 decision 3 / #1349: the resolved node and its tree, for record-time evidence (absent for presence-only predicates). */ + node?: SnapshotNode; + preActionNodes?: SnapshotNode[]; +}; + +export type IsSelectorCommandOptions = CommandContext & + SelectorSnapshotOptions & { + target: SelectorTarget; + }; + +export const isCommand: RuntimeCommand = async ( + runtime, + options, +): Promise => { + const admitted = checkIsPredicate(options.predicate); + if (!admitted.ok) throw new AppError(admitted.code, admitted.message, { hint: admitted.hint }); + const predicate = admitted.predicate; + if (predicate === 'absent') { + const refusedOption = absenceCaptureOptionRefusal(options); + if (refusedOption) { + throw absenceCaptureOptionError(refusedOption); + } + } + if (predicate === 'text' && !options.expectedText) { + throw new AppError('INVALID_ARGS', IS_TEXT_VALUE_REQUIRED_MESSAGE); + } + const selectorExpression = options.selector; + const capture = await captureIsSnapshot(runtime, options, predicate, selectorExpression); + if (predicate === 'exists') + return await resolveExistsPredicate(runtime, capture, selectorExpression); + if (predicate === 'absent') { + return await resolveAbsenceObservation( + capture.snapshot, + selectorExpression, + runtime.backend.platform, + ); + } + return await resolveAssertedPredicate(runtime, options, capture, predicate, selectorExpression); +}; + +async function captureIsSnapshot( + runtime: AgentDeviceRuntime, + options: IsCommandOptions, + predicate: IsPredicate, + selectorExpression: string, +): Promise { + try { + return await captureSelectorSnapshot(runtime, options, { + updateSession: true, + ...deriveSelectorCapturePolicy(predicate), + }); + } catch (error) { + if (predicate !== 'absent' || isRequestCanceledError(error)) throw error; + throw absenceUnreadableError(selectorExpression, error); + } +} + +async function resolveExistsPredicate( + runtime: AgentDeviceRuntime, + capture: CapturedSnapshot, + selectorExpression: string, +): Promise { + const matched = await resolveSelectorPipeline( + SELECTOR_PIPELINE_POLICIES.readAny, + capture.snapshot.nodes, + selectorExpression, + { platform: runtime.backend.platform }, + ); + if (matched.kind !== 'target') { + throw new AppError( + 'COMMAND_FAILED', + formatSelectorFailure(selectorExpression, [], { unique: false }), + { + hint: selectorFailureHint([]), + }, + ); + } + return { + predicate: 'exists', + pass: true, + selector: matched.selector, + matches: matched.matches, + selectorChain: readSelectorAlternatives(selectorExpression), + }; +} + +async function resolveAssertedPredicate( + runtime: AgentDeviceRuntime, + options: IsCommandOptions, + capture: CapturedSnapshot, + predicate: Exclude, + selectorExpression: string, +): Promise { + const outcome = await resolveSelectorPipeline( + SELECTOR_PIPELINE_POLICIES.readUnique, + capture.snapshot.nodes, + selectorExpression, + { platform: runtime.backend.platform }, + { + onResolved: (node, nodes) => + assertExpectedResolvedTarget(node, nodes, options.expectedResolvedTarget, 'is'), + }, + ); + if (outcome.kind !== 'target') { + throw new AppError( + 'COMMAND_FAILED', + formatSelectorFailure(selectorExpression, [], { unique: true }), + { + command: 'is', + reason: INTERACTION_ERROR_REASONS.selectorNotFound, + predicate: predicate, + selector: selectorExpression, + hint: selectorFailureHint([]), + }, + ); + } + const result = evaluateIsPredicate({ + predicate, + node: outcome.node, + nodes: capture.snapshot.nodes, + expectedText: options.expectedText, + platform: runtime.backend.platform, + }); + if (!result.pass) { + throw new AppError( + 'COMMAND_FAILED', + `is ${predicate} failed for selector ${outcome.selector}: ${result.details}`, + { + command: 'is', + reason: INTERACTION_ERROR_REASONS.predicateFailed, + predicate: predicate, + selector: outcome.selector, + predicateDetails: result.details, + }, + ); + } + return { + predicate, + pass: true, + selector: outcome.selector, + ...(predicate === 'text' ? { text: result.actualText } : {}), + selectorChain: readSelectorAlternatives(selectorExpression), + node: outcome.node, + preActionNodes: capture.snapshot.nodes, + }; +} + +export const isVisibleCommand: RuntimeCommand = async ( + runtime, + options, +): Promise => + await isCommand(runtime, { + ...options, + predicate: 'visible', + selector: options.target.selector, + }); + +export const isHiddenCommand: RuntimeCommand = async ( + runtime, + options, +): Promise => + await isCommand(runtime, { + ...options, + predicate: 'hidden', + selector: options.target.selector, + }); diff --git a/src/commands/interaction/runtime/selector-read.test.ts b/src/commands/interaction/runtime/selector-read.test.ts index 72d73a160a..6f8566e753 100644 --- a/src/commands/interaction/runtime/selector-read.test.ts +++ b/src/commands/interaction/runtime/selector-read.test.ts @@ -116,410 +116,6 @@ test('runtime selectors pass runtime signal to backend snapshot capture', async assert.equal(signal, controller.signal); }); -test('runtime selectors forward public snapshot options to backend capture', async () => { - const snapshot = selectorReadSnapshot(); - let captureOptions: BackendSnapshotOptions | undefined; - const device = createAgentDevice({ - backend: { - platform: 'ios', - captureSnapshot: async (_context, options) => { - captureOptions = options; - return { snapshot }; - }, - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([{ name: 'default', snapshot }]), - policy: localCommandPolicy(), - }); - - await device.selectors.is({ - session: 'default', - predicate: 'exists', - selector: 'label=Continue', - depth: 2, - scope: 'Login', - raw: true, - }); - - assert.deepEqual(captureOptions, { - interactiveOnly: false, - depth: 2, - scope: 'Login', - raw: true, - includeRects: false, - }); -}); - -test('runtime visibility predicates request snapshot rects', async () => { - const snapshot = selectorReadSnapshot(); - let captureOptions: BackendSnapshotOptions | undefined; - const device = createAgentDevice({ - backend: { - platform: 'web', - captureSnapshot: async (_context, options) => { - captureOptions = options; - return { snapshot }; - }, - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([{ name: 'default', snapshot }]), - policy: localCommandPolicy(), - }); - - await device.selectors.isVisible(selector('label=Continue'), { - session: 'default', - }); - - assert.equal(captureOptions?.includeRects, true); -}); - -test('runtime focused predicate requests a full snapshot', async () => { - const snapshot = makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'Cell', - label: 'Profiles and Accounts', - focused: true, - }, - ]); - let captureOptions: BackendSnapshotOptions | undefined; - const device = createAgentDevice({ - backend: { - platform: 'ios', - captureSnapshot: async (_context, options) => { - captureOptions = options; - return { snapshot }; - }, - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([{ name: 'default', snapshot }]), - policy: localCommandPolicy(), - }); - - const result = await device.selectors.is({ - session: 'default', - predicate: 'focused', - selector: 'role=cell label="Profiles and Accounts"', - }); - - assert.equal(result.pass, true); - assert.equal(captureOptions?.interactiveOnly, false); -}); - -test('runtime focused predicate reads focused Android TV nodes from the full tree', async () => { - const fullSnapshot = makeSnapshotState( - [ - { - index: 0, - depth: 0, - type: 'TextView', - label: 'Featured', - focused: true, - hittable: false, - }, - ], - { backend: 'android' }, - ); - const interactiveSnapshot = makeSnapshotState([], { backend: 'android' }); - let captureOptions: BackendSnapshotOptions | undefined; - const device = createAgentDevice({ - backend: { - platform: 'android', - captureSnapshot: async (_context, options) => { - captureOptions = options; - return { snapshot: options?.interactiveOnly ? interactiveSnapshot : fullSnapshot }; - }, - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([{ name: 'default', snapshot: interactiveSnapshot }]), - policy: localCommandPolicy(), - }); - - const result = await device.selectors.is({ - session: 'default', - predicate: 'focused', - selector: 'label=Featured', - }); - - assert.equal(result.pass, true); - assert.equal(captureOptions?.interactiveOnly, false); -}); - -test('runtime is validates selector predicates', async () => { - const device = createSelectorDevice(selectorReadSnapshot()); - - const result = await device.selectors.is({ - session: 'default', - predicate: 'exists', - selector: 'label=Continue', - }); - - assert.deepEqual(result, { - predicate: 'exists', - pass: true, - selector: 'label=Continue', - matches: 1, - selectorChain: ['label=Continue'], - }); -}); - -test('runtime is absent passes when the selector has no match', async () => { - const device = createSelectorDevice( - makeSnapshotState([{ index: 0, depth: 0, type: 'StaticText', label: 'Current screen' }]), - ); - - const result = await device.selectors.is({ - session: 'default', - predicate: 'absent', - selector: 'label="Removed row"', - }); - - assert.equal(result.predicate, 'absent'); - assert.equal(result.pass, true); -}); - -test('runtime is absent uses one readAny capture without requesting rects', async () => { - const snapshot = makeSnapshotState([ - { index: 0, depth: 0, type: 'StaticText', label: 'Current screen' }, - ]); - let captures = 0; - let captureOptions: BackendSnapshotOptions | undefined; - const device = createAgentDevice({ - backend: { - platform: 'ios', - captureSnapshot: async (_context, options) => { - captures += 1; - captureOptions = options; - return { snapshot }; - }, - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions: createMemorySessionStore([{ name: 'default', snapshot }]), - policy: localCommandPolicy(), - }); - - const result = await device.selectors.is({ - session: 'default', - predicate: 'absent', - selector: 'label="Removed row"', - }); - - assert.deepEqual(result, { - predicate: 'absent', - pass: true, - selector: 'label="Removed row"', - matches: 0, - }); - assert.equal(captures, 1); - assert.equal(captureOptions?.includeRects, false); - assert.equal(captureOptions?.depth, undefined); - assert.equal(captureOptions?.scope, undefined); -}); - -test('runtime is absent reports one matching node without visibility or geometry claims', async () => { - const snapshot = makeSnapshotState([ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeButton', - identifier: 'save', - label: 'Save', - visibleToUser: false, - }, - ]); - const device = createSelectorDevice(snapshot); - - await assert.rejects( - device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Save"' }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.code, 'COMMAND_FAILED'); - assert.deepEqual(error.details, { - command: 'is', - reason: 'predicate_failed', - predicate: 'absent', - selector: 'label="Save"', - matches: 1, - observation: 'present', - firstMatch: { id: 'save', role: 'button', label: 'Save', text: 'Save' }, - }); - assert.equal(/visible|hidden|rect/i.test(error.message), false); - return true; - }, - ); -}); - -test('runtime is absent reports the match count and refinement hint for multiple matches', async () => { - const snapshot = makeSnapshotState([ - { index: 0, depth: 0, type: 'Button', identifier: 'save-one', label: 'Save' }, - { index: 1, depth: 0, type: 'Button', identifier: 'save-two', label: 'Save' }, - ]); - const device = createSelectorDevice(snapshot); - - await assert.rejects( - device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Save"' }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.code, 'COMMAND_FAILED'); - assert.equal(error.details?.reason, 'predicate_failed'); - assert.equal(error.details?.matches, 2); - assert.deepEqual(error.details?.firstMatch, { - id: 'save-one', - role: 'button', - label: 'Save', - text: 'Save', - }); - assert.equal(error.details?.hint, 'Refine the selector to match no elements.'); - return true; - }, - ); -}); - -test('runtime is absent keeps the first matching selector alternative as the readAny domain', async () => { - const snapshot = makeSnapshotState([ - { index: 0, depth: 0, type: 'Button', identifier: 'save-one', label: 'Save' }, - { index: 1, depth: 0, type: 'Button', identifier: 'save-two', label: 'Save' }, - { index: 2, depth: 0, type: 'Button', identifier: 'gone', label: 'Gone' }, - ]); - const device = createSelectorDevice(snapshot); - - await assert.rejects( - device.selectors.is({ - session: 'default', - predicate: 'absent', - selector: 'label="Save" || label="Gone"', - }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.code, 'COMMAND_FAILED'); - assert.equal(error.details?.reason, 'predicate_failed'); - assert.equal(error.details?.observation, 'present'); - assert.equal(error.details?.matches, 2); - assert.deepEqual(error.details?.firstMatch, { - id: 'save-one', - role: 'button', - label: 'Save', - text: 'Save', - }); - return true; - }, - ); -}); - -test('runtime is absent fails closed for a sparse capture and preserves the last session snapshot', async () => { - const initial = makeSnapshotState([{ index: 0, depth: 0, type: 'StaticText', label: 'Initial' }]); - const sparse = makeSnapshotState([], { - snapshotQuality: { - state: 'sparse', - backend: 'private-ax', - reason: 'sparse tree', - reasonCode: 'sparse-tree', - }, - }); - const sessions = createMemorySessionStore([{ name: 'default', snapshot: initial }]); - const device = createAgentDevice({ - backend: { - platform: 'ios', - captureSnapshot: async () => ({ snapshot: sparse }), - } satisfies AgentDeviceBackend, - artifacts: createLocalArtifactAdapter(), - sessions, - policy: localCommandPolicy(), - }); - - await assert.rejects( - device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Gone"' }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.code, 'COMMAND_FAILED'); - assert.equal(error.details?.reason, 'predicate_failed'); - assert.equal(error.details?.observation, 'sparse'); - assert.equal(error.details?.matches, 0); - assert.deepEqual(error.details?.snapshotQuality, sparse.snapshotQuality); - return true; - }, - ); - assert.equal((await sessions.get('default'))?.snapshot?.nodes[0]?.label, 'Initial'); -}); - -test('runtime is absent fails closed for a truncated capture even with zero matches', async () => { - const snapshot = makeSnapshotState([]); - const device = createSelectorDevice(snapshot, { - captureSnapshot: async () => ({ snapshot, truncated: true }), - }); - - await assert.rejects( - device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Gone"' }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.code, 'COMMAND_FAILED'); - assert.equal(error.details?.reason, 'predicate_failed'); - assert.equal(error.details?.observation, 'truncated'); - assert.equal(error.details?.matches, 0); - assert.equal(error.details?.truncated, true); - return true; - }, - ); -}); - -test('runtime is absent maps an unreadable capture to typed predicate failure evidence', async () => { - const device = createSelectorDevice(makeSnapshotState([]), { - captureSnapshot: async () => { - throw new AppError('COMMAND_FAILED', 'accessibility capture failed', { - reason: 'capture-failed', - }); - }, - }); - - await assert.rejects( - device.selectors.is({ session: 'default', predicate: 'absent', selector: 'label="Gone"' }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.code, 'COMMAND_FAILED'); - assert.equal(error.details?.reason, 'predicate_failed'); - assert.equal(error.details?.observation, 'unreadable'); - assert.equal(error.details?.captureErrorCode, 'COMMAND_FAILED'); - assert.equal(error.details?.captureErrorReason, 'capture-failed'); - assert.equal(error.details?.matches, 0); - return true; - }, - ); -}); - -test('runtime is absent refuses depth and scope before capture with typed invalid arguments', async () => { - let captures = 0; - const device = createSelectorDevice(makeSnapshotState([]), { - captureSnapshot: async () => { - captures += 1; - return { snapshot: makeSnapshotState([]) }; - }, - }); - - for (const [option, value] of [ - ['scope', 'Login'], - ['depth', 2], - ] as const) { - await assert.rejects( - device.selectors.is({ - session: 'default', - predicate: 'absent', - selector: 'label="Gone"', - [option]: value, - }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.code, 'INVALID_ARGS'); - assert.equal(error.details?.command, 'is'); - assert.equal(error.details?.predicate, 'absent'); - assert.equal(error.details?.rejectedOption, option); - return true; - }, - ); - } - assert.equal(captures, 0); -}); - test('runtime find get_text reads the matched node', async () => { const device = createSelectorDevice(selectorReadSnapshot(), { readText: 'Continue', @@ -1026,54 +622,3 @@ test('runtime wait fails immediately on a helper MECHANISM failure even though i ); assert.equal(attempts(), 1); }); - -// Regression: admission normalizes a predicate's case, and every branch below it has to read -// the ADMITTED value. Reading `options.predicate` instead let an uppercase predicate past the -// gate and then evaluated it against lower-case branches — `EXISTS` skipped its own branch and -// `TEXT` compared nothing — so the command answered wrongly instead of refusing or working. -test('runtime is admits an upper-case predicate and evaluates it as the normalized one', async () => { - const snapshot = makeSnapshotState([ - { index: 0, depth: 0, type: 'StaticText', label: 'Greeting' }, - ]); - const device = createSelectorDevice(snapshot); - - const exists = await device.selectors.is({ - session: 'default', - predicate: 'EXISTS' as 'exists', - selector: 'label=Greeting', - }); - assert.equal(exists.predicate, 'exists'); - assert.equal(exists.pass, true); - - const text = await device.selectors.is({ - session: 'default', - predicate: 'TEXT' as 'text', - selector: 'label=Greeting', - expectedText: 'Greeting', - }); - assert.equal(text.predicate, 'text'); - assert.equal(text.pass, true); - assert.equal(text.text, 'Greeting'); -}); - -test('runtime is still refuses a predicate that is not in the vocabulary', async () => { - const device = createSelectorDevice( - makeSnapshotState([{ index: 0, depth: 0, type: 'StaticText', label: 'Greeting' }]), - ); - - await assert.rejects( - async () => - await device.selectors.is({ - session: 'default', - predicate: 'shiny' as 'exists', - selector: 'label=Greeting', - }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.code, 'INVALID_ARGS'); - // ADR 0010: the refusal carries recovery guidance on every surface, not just the daemon's. - assert.match(String(error.details?.hint ?? ''), /is /); - return true; - }, - ); -}); diff --git a/src/commands/interaction/runtime/selector-read.ts b/src/commands/interaction/runtime/selector-read.ts index 1f697da574..6f66df4a54 100644 --- a/src/commands/interaction/runtime/selector-read.ts +++ b/src/commands/interaction/runtime/selector-read.ts @@ -4,14 +4,9 @@ import { formatSelectorFailure, selectorFailureHint, buildSelectorChainForNode, - checkIsPredicate, - evaluateIsPredicate, - IS_TEXT_VALUE_REQUIRED_MESSAGE, - readSelectorAlternatives, parseFindSelectorExpression, type FindAction, type FindLocator, - type IsPredicate, } from '@agent-device/selectors'; import { listSelectorPipelineMatches, @@ -22,14 +17,12 @@ import { SELECTOR_PIPELINE_POLICIES } from '../../../core/selector-pipeline-poli import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; -import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; +import { AppError } from '@agent-device/kernel/errors'; import type { ElementTarget, FindReadResult, ResolvedTarget, - SelectorTarget, } from '@agent-device/contracts/interaction'; -import { INTERACTION_ERROR_REASONS } from '@agent-device/contracts/interaction-error'; import type { RuntimeCommand } from '../../runtime-types.ts'; import { assertExpectedResolvedTarget, type ExpectedResolvedTarget } from './resolution.ts'; import { @@ -42,12 +35,6 @@ import { } from './selector-read-shared.ts'; import { findSnapshotScope, sparseSelectorSnapshotError } from './selector-read-utils.ts'; import { deriveSelectorCapturePolicy } from './selector-capture-policy.ts'; -import { absenceCaptureOptionRefusal } from '../../../core/absence-observation.ts'; -import { - absenceCaptureOptionError, - absenceUnreadableError, -} from '../../../core/absence-observation-errors.ts'; -import { resolveAbsenceObservation } from '../../../core/absence-observation-resolution.ts'; import { createWaitPolling, type WaitPollDeadline, waitTimeoutError } from './wait-polling.ts'; import { createSelectorWaitCommands, @@ -68,7 +55,7 @@ export type { WaitCommandResult, WaitForTextCommandOptions, } from './selector-wait.ts'; -export type { ElementTarget, ResolvedTarget, SelectorTarget }; +export type { ElementTarget, ResolvedTarget }; export type FindReadCommandOptions = CommandContext & { locator?: FindLocator; @@ -116,32 +103,6 @@ export type GetAttrsCommandOptions = CommandContext & target: ElementTarget; }; -export type IsCommandOptions = CommandContext & - SelectorSnapshotOptions & { - predicate: IsPredicate; - selector: string; - expectedText?: string; - /** ADR 0012 step 4: replay-only post-resolution guard; see resolution.ts. */ - expectedResolvedTarget?: ExpectedResolvedTarget; - }; - -export type IsCommandResult = { - predicate: IsCommandOptions['predicate']; - pass: true; - selector: string; - matches?: number; - text?: string; - selectorChain?: string[]; - /** ADR 0012 decision 3 / #1349: the resolved node and its tree, for record-time evidence (absent for presence-only predicates). */ - node?: SnapshotNode; - preActionNodes?: SnapshotNode[]; -}; - -export type IsSelectorCommandOptions = CommandContext & - SelectorSnapshotOptions & { - target: SelectorTarget; - }; - const selectorWaitCommands = createSelectorWaitCommands({ captureSnapshot: captureSelectorSnapshot, requireSnapshot: requireSnapshotSession, @@ -286,163 +247,6 @@ export const getAttrsCommand: RuntimeCommand< return result; }; -export const isCommand: RuntimeCommand = async ( - runtime, - options, -): Promise => { - const admitted = checkIsPredicate(options.predicate); - if (!admitted.ok) throw new AppError(admitted.code, admitted.message, { hint: admitted.hint }); - const predicate = admitted.predicate; - if (predicate === 'absent') { - const refusedOption = absenceCaptureOptionRefusal(options); - if (refusedOption) { - throw absenceCaptureOptionError(refusedOption); - } - } - if (predicate === 'text' && !options.expectedText) { - throw new AppError('INVALID_ARGS', IS_TEXT_VALUE_REQUIRED_MESSAGE); - } - const selectorExpression = options.selector; - const capture = await captureIsSnapshot(runtime, options, predicate, selectorExpression); - if (predicate === 'exists') - return await resolveExistsPredicate(runtime, capture, selectorExpression); - if (predicate === 'absent') { - return await resolveAbsenceObservation( - capture.snapshot, - selectorExpression, - runtime.backend.platform, - ); - } - return await resolveAssertedPredicate(runtime, options, capture, predicate, selectorExpression); -}; - -async function captureIsSnapshot( - runtime: AgentDeviceRuntime, - options: IsCommandOptions, - predicate: IsPredicate, - selectorExpression: string, -): Promise { - try { - return await captureSelectorSnapshot(runtime, options, { - updateSession: true, - ...deriveSelectorCapturePolicy(predicate), - }); - } catch (error) { - if (predicate !== 'absent' || isRequestCanceledError(error)) throw error; - throw absenceUnreadableError(selectorExpression, error); - } -} - -async function resolveExistsPredicate( - runtime: AgentDeviceRuntime, - capture: CapturedSnapshot, - selectorExpression: string, -): Promise { - const matched = await resolveSelectorPipeline( - SELECTOR_PIPELINE_POLICIES.readAny, - capture.snapshot.nodes, - selectorExpression, - { platform: runtime.backend.platform }, - ); - if (matched.kind !== 'target') { - throw new AppError( - 'COMMAND_FAILED', - formatSelectorFailure(selectorExpression, [], { unique: false }), - { - hint: selectorFailureHint([]), - }, - ); - } - return { - predicate: 'exists', - pass: true, - selector: matched.selector, - matches: matched.matches, - selectorChain: readSelectorAlternatives(selectorExpression), - }; -} - -async function resolveAssertedPredicate( - runtime: AgentDeviceRuntime, - options: IsCommandOptions, - capture: CapturedSnapshot, - predicate: Exclude, - selectorExpression: string, -): Promise { - const outcome = await resolveSelectorPipeline( - SELECTOR_PIPELINE_POLICIES.readUnique, - capture.snapshot.nodes, - selectorExpression, - { platform: runtime.backend.platform }, - { - onResolved: (node, nodes) => - assertExpectedResolvedTarget(node, nodes, options.expectedResolvedTarget, 'is'), - }, - ); - if (outcome.kind !== 'target') { - throw new AppError( - 'COMMAND_FAILED', - formatSelectorFailure(selectorExpression, [], { unique: true }), - { - command: 'is', - reason: INTERACTION_ERROR_REASONS.selectorNotFound, - predicate: predicate, - selector: selectorExpression, - hint: selectorFailureHint([]), - }, - ); - } - const result = evaluateIsPredicate({ - predicate, - node: outcome.node, - nodes: capture.snapshot.nodes, - expectedText: options.expectedText, - platform: runtime.backend.platform, - }); - if (!result.pass) { - throw new AppError( - 'COMMAND_FAILED', - `is ${predicate} failed for selector ${outcome.selector}: ${result.details}`, - { - command: 'is', - reason: INTERACTION_ERROR_REASONS.predicateFailed, - predicate: predicate, - selector: outcome.selector, - predicateDetails: result.details, - }, - ); - } - return { - predicate, - pass: true, - selector: outcome.selector, - ...(predicate === 'text' ? { text: result.actualText } : {}), - selectorChain: readSelectorAlternatives(selectorExpression), - node: outcome.node, - preActionNodes: capture.snapshot.nodes, - }; -} - -export const isVisibleCommand: RuntimeCommand = async ( - runtime, - options, -): Promise => - await isCommand(runtime, { - ...options, - predicate: 'visible', - selector: options.target.selector, - }); - -export const isHiddenCommand: RuntimeCommand = async ( - runtime, - options, -): Promise => - await isCommand(runtime, { - ...options, - predicate: 'hidden', - selector: options.target.selector, - }); - async function waitForFindMatch( runtime: AgentDeviceRuntime, options: FindReadCommandOptions, diff --git a/src/commands/interaction/selectors.ts b/src/commands/interaction/selectors.ts index caddfa6c69..973f305bf6 100644 --- a/src/commands/interaction/selectors.ts +++ b/src/commands/interaction/selectors.ts @@ -8,8 +8,6 @@ import { normalizeIsPositionals, UNSUPPORTED_FIND_ACTION_HINT, } from '@agent-device/selectors'; -import { absenceCaptureOptionRefusal } from '../../core/absence-observation.ts'; -import { absenceCaptureOptionError } from '../../core/absence-observation-errors.ts'; import { direct, optionalCliNumber, @@ -156,6 +154,29 @@ function readIsOptionsFromPositionals(positionals: string[], flags: CliFlags): I return { ...base, predicate, selector: split.selectorExpression }; } +type AbsenceCaptureOption = 'depth' | 'scope'; + +function absenceCaptureOptionRefusal(options: { + depth?: number; + scope?: string; +}): AbsenceCaptureOption | undefined { + if (options.scope !== undefined) return 'scope'; + if (options.depth !== undefined) return 'depth'; + return undefined; +} + +function absenceCaptureOptionError(option: AbsenceCaptureOption): AppError { + const message = + option === 'scope' + ? 'is absent does not support --scope; it requires an unscoped capture' + : 'is absent does not support --depth; it requires a full-depth capture'; + return new AppError('INVALID_ARGS', message, { + command: 'is', + predicate: 'absent', + rejectedOption: option, + }); +} + function readFindLocator(value: string | undefined): FindOptions['locator'] | undefined { if ( value === 'text' || diff --git a/src/core/absence-observation.test.ts b/src/core/absence-observation.test.ts index fb55806ab5..8021027b5f 100644 --- a/src/core/absence-observation.test.ts +++ b/src/core/absence-observation.test.ts @@ -33,12 +33,26 @@ test('classifies matches as present and exposes only stable first-match fields', assert.deepEqual(observation, { kind: 'present', matches: 2, - firstMatch: { id: 'save', role: 'button', label: 'Save', text: 'Save' }, + firstMatch: { id: 'save', role: 'button', label: 'Save' }, }); assert.equal('rect' in observation.firstMatch, false); assert.equal('visibleToUser' in observation.firstMatch, false); }); +test('bounds optional first-match text by UTF-8 bytes', () => { + const snapshot = makeSnapshotState([ + { index: 0, type: 'Button', value: '\u{1F642}'.repeat(100) }, + ]); + + const observation = classifyAbsenceObservation(snapshot, [snapshot.nodes[0]!]); + + assert.equal(observation.kind, 'present'); + if (observation.kind === 'present') { + assert.equal(Buffer.byteLength(observation.firstMatch.text ?? '', 'utf8') <= 256, true); + assert.equal(observation.firstMatch.text, '\u{1F642}'.repeat(64)); + } +}); + test('classifies sparse and truncated captures before evaluating zero matches', () => { const sparse = classifyAbsenceObservation( makeSnapshotState([], { diff --git a/src/core/absence-observation.ts b/src/core/absence-observation.ts index 14890825af..e3a5b0cc76 100644 --- a/src/core/absence-observation.ts +++ b/src/core/absence-observation.ts @@ -1,5 +1,9 @@ import { extractNodeText, normalizeType } from '@agent-device/contracts/snapshot'; -import { readNodeLocalIdentity } from '@agent-device/ad-script'; +import { + readNodeLocalIdentity, + TARGET_ANNOTATION_MAX_FIELD_BYTES, + truncateToUtf8Bytes, +} from '@agent-device/ad-script'; import type { SnapshotNode, SnapshotQualityVerdict, @@ -103,9 +107,9 @@ export function isLegacySparseIosInteractiveSnapshot( function stableFirstMatch(node: SnapshotNode): AbsenceFirstMatch { const identity = readNodeLocalIdentity(node); - const text = extractNodeText(node); + const text = truncateToUtf8Bytes(extractNodeText(node), TARGET_ANNOTATION_MAX_FIELD_BYTES); return { ...identity, - ...(text ? { text } : {}), + ...(text && text !== identity.label ? { text } : {}), }; } diff --git a/src/daemon/__tests__/is-runtime.test.ts b/src/daemon/__tests__/is-runtime.test.ts index 9f840f4f90..38a0bde852 100644 --- a/src/daemon/__tests__/is-runtime.test.ts +++ b/src/daemon/__tests__/is-runtime.test.ts @@ -1,5 +1,6 @@ import { beforeEach, expect, test, vi } from 'vitest'; import type { SnapshotResult } from '@agent-device/contracts/snapshot-runtime'; +import { buildSnapshotPresentationKey } from '@agent-device/kernel/snapshot'; import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; import { makeAndroidSession, @@ -8,6 +9,7 @@ import { } from '../../__tests__/test-utils/session-factories.ts'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; import { withTestDeviceInventory } from '../../__tests__/test-utils/device-inventory-gateways.ts'; +import { makeSnapshotState } from '../../__tests__/test-utils/snapshot-builders.ts'; import type { DaemonRequest } from '../types.ts'; import { selectorCaptureFixture } from './selector-capture-fixture.ts'; @@ -108,6 +110,48 @@ test('is absent uses the bound readAny capture for selector-first input without expect(fixture.captures[0]?.options?.includeRects).toBe(false); }); +test('is absent bypasses a cached no-match snapshot before evaluating the bound capture', async () => { + const cachedAbsent = { + ...makeSnapshotState([{ index: 0, type: 'StaticText', label: 'Current screen' }], { + backend: 'xctest', + producer: 'apple-runner', + }), + presentationKey: buildSnapshotPresentationKey({}), + } satisfies NonNullable['snapshot']>; + const fixture = selectorCaptureFixture({ + snapshot: () => ({ + nodes: [{ index: 0, type: 'Button', label: 'Removed row' }], + backend: 'xctest', + producer: 'apple-runner', + }), + }); + const sessionStore = makeSessionStore(); + sessionStore.set( + 'is-absent-fresh', + makeIosAppSession('is-absent-fresh', { snapshot: cachedAbsent }), + ); + + const response = await dispatchIsViaRuntime({ + req: isRequest('is-absent-fresh', ['absent', 'label="Removed row"']), + sessionName: 'is-absent-fresh', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice: fixture.bindDevice, + }); + + expect(response?.ok).toBe(false); + if (response?.ok === false) { + expect(response.error.details).toMatchObject({ + command: 'is', + reason: 'predicate_failed', + predicate: 'absent', + matches: 1, + observation: 'present', + }); + } + expect(fixture.captures).toHaveLength(1); +}); + test('is absent fails closed for a quality-less legacy iOS root-only capture', async () => { const fixture = selectorCaptureFixture({ snapshot: () => ({ diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index 408b528d8c..844f6b2195 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -28,6 +28,7 @@ import type { AndroidObservationAdapter } from '@agent-device/contracts/android- import type { PlatformResourceCleanup } from '@agent-device/contracts/platform-resource-cleanup'; import { getRequestSignal } from '@agent-device/host-kit/request'; import { snapshotOptionsToFlags } from '../backend-snapshot-options.ts'; +import { checkIsArgs } from '@agent-device/selectors'; export type SelectorRuntimeParams = { req: DaemonRequest; @@ -184,6 +185,7 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice const needsFreshSnapshot = req.command === 'wait' || req.command === 'find' || + isAbsentPredicateRequest(req) || (includeRects && device.platform === 'web'); return await captureRuntime.capture({ flags, @@ -228,3 +230,9 @@ function createSelectorBackend(params: SelectorRuntimeDeviceParams): AgentDevice : {}), }; } + +function isAbsentPredicateRequest(req: DaemonRequest): boolean { + if (req.command !== 'is') return false; + const checked = checkIsArgs(req.positionals ?? []); + return checked.ok && checked.predicate === 'absent'; +} diff --git a/test/integration/android-emulator-e2e/coverage-manifest.ts b/test/integration/android-emulator-e2e/coverage-manifest.ts index fe47b1a7f1..cdd99c728e 100644 --- a/test/integration/android-emulator-e2e/coverage-manifest.ts +++ b/test/integration/android-emulator-e2e/coverage-manifest.ts @@ -114,7 +114,10 @@ export const ANDROID_EMULATOR_E2E_COVERAGE = { ANDROID_INSTALL_SOURCE_CONTRACT_EVIDENCE, 'Android install-source resolves an installable artifact with typed identity', ), - [C.is]: live('smoke:automation-system', 'visible predicate passes for Android fixture node'), + [C.is]: live( + 'smoke:automation-system', + 'visible predicate passes and absent observes the unmounted Android modal control', + ), [C.keyboard]: live('smoke:keyboard-ime', 'safe dismissal hides keyboard without navigating Back'), [C.logs]: live( 'full:observability-artifacts', diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.ts b/test/integration/android-emulator-e2e/live-automation-scenario.ts index 4d8016ecfc..64f1138986 100644 --- a/test/integration/android-emulator-e2e/live-automation-scenario.ts +++ b/test/integration/android-emulator-e2e/live-automation-scenario.ts @@ -104,6 +104,13 @@ export async function assertAutomationSystem(context: LiveContext): Promise await assertWaitText(context, 'Automation sheet'); await runStep(context, 'close fixture sheet', ['click', 'id="automation-close-sheet"']); await assertWaitText(context, 'Automation lab'); + const absentSheet = await runStep(context, 'assert closed fixture sheet is absent', [ + 'is', + 'absent', + 'id="automation-close-sheet"', + ]); + assert.equal(absentSheet.json?.data?.pass, true, JSON.stringify(absentSheet.json)); + verifyCommand(context, C.is, 'strict absence observes the unmounted fixture sheet control'); await runStep(context, 'restore automation route top after sheet', ['scroll', 'top']); verifyBehavior( context,