refactor(mcp): extract reference-pin state into tool-ref-pins module (#1344) (#1345)

* refactor(mcp): extract ref-pin state into focused tool-ref-pins module

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(mcp): add success-path ref-pin wiring test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(mcp): narrow tool-ref-pins result types and replace as-cast with type guard

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(mcp): remove unnecessary as-casts in tool-ref-pins and command-tools

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(mcp): align ref-pin result handling with #1343 typed result projection

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(mcp): upstream types for ref-pin module (#1345)

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(mcp): introduce honest public targetKind-discriminated interaction response contracts

Replace the public AgentDeviceClient return types for press/click/fill/longpress/find with serialized-payload-shaped response data (targetKind, flat ref/selector/x/y, per-command extras) instead of internal runtime result types. The internal PressCommandResult/FillCommandResult/LongPressCommandResult keep their kind/target shapes for the daemon runtime; the public CommandResultMap now points to the new response contracts.

- Add PressCommandResponseData/FillCommandResponseData/LongPressCommandResponseData/FindCommandResponseData in src/contracts/interaction.ts.
- Update CommandResultMap and command-result tests.
- Add client-facing shape tests asserting the public response data discriminates on targetKind and exposes flat identity fields.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(contracts): include cost and iOS Maestro fallback fields in public response contracts

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Michał Pierzchała <thymikee@gmail.com>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

Author: devin-ai-integration[bot]

src/__tests__/client.test.ts

@@ -3,6 +3,13 @@ import assert from 'node:assert/strict';
 import { mkdtempSync } from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import type {
+  ClickCommandResponseData,
+  FillCommandResponseData,
+  FindCommandResponseData,
+  LongPressCommandResponseData,
+  PressCommandResponseData,
+} from '../contracts/interaction.ts';
 import {
   createAgentDeviceClient,
   type AgentDeviceClient,
@@ -1140,3 +1147,178 @@ test('capture.snapshot passes a digest (non-default level) payload through unnor
   assert.equal(asRecord.nodeCount, 3);
   assert.ok(!('identifiers' in asRecord));
 });
+
+test('interactions expose targetKind-discriminated public response data', async () => {
+  const setup = createTransport(async (req) => {
+    if (req.command === 'press') {
+      return {
+        ok: true,
+        data: {
+          targetKind: 'ref',
+          ref: 'e5',
+          x: 88,
+          y: 99,
+          message: 'Tapped @e5 (88, 99)',
+        },
+      };
+    }
+    if (req.command === 'click') {
+      return {
+        ok: true,
+        data: {
+          targetKind: 'point',
+          x: 10,
+          y: 20,
+          button: 'secondary',
+          message: 'Tapped (10, 20)',
+        },
+      };
+    }
+    if (req.command === 'fill') {
+      return {
+        ok: true,
+        data: {
+          targetKind: 'ref',
+          ref: 'e5',
+          x: 88,
+          y: 99,
+          text: 'hello',
+          message: 'Filled 5 chars',
+        },
+      };
+    }
+    if (req.command === 'longpress') {
+      return {
+        ok: true,
+        data: {
+          targetKind: 'selector',
+          selector: 'label=Foo',
+          x: 30,
+          y: 40,
+          gesture: 'longpress',
+          durationMs: 500,
+          message: 'Long pressed label=Foo (30, 40)',
+        },
+      };
+    }
+    if (req.command === 'find') {
+      return {
+        ok: true,
+        data: { ref: '@e5', refsGeneration: 42, text: 'Hello' },
+      };
+    }
+    throw new Error(`unexpected command: ${req.command}`);
+  });
+  const client = createAgentDeviceClient(setup.config, { transport: setup.transport });
+
+  const press = await client.interactions.press({ ref: '@e5' });
+  const click = await client.interactions.click({ x: 10, y: 20, button: 'secondary' });
+  const fill = await client.interactions.fill({ ref: '@e5', text: 'hello' });
+  const longPress = await client.interactions.longPress({
+    selector: 'label=Foo',
+    durationMs: 500,
+  });
+  const find = await client.interactions.find({
+    locator: 'label',
+    query: 'Foo',
+    action: 'getText',
+  });
+
+  const pressType: Equal<typeof press, PressCommandResponseData> = true;
+  const clickType: Equal<typeof click, ClickCommandResponseData> = true;
+  const fillType: Equal<typeof fill, FillCommandResponseData> = true;
+  const longPressType: Equal<typeof longPress, LongPressCommandResponseData> = true;
+  const findType: Equal<typeof find, FindCommandResponseData> = true;
+
+  assert.equal(press.targetKind, 'ref');
+  assert.equal(press.ref, 'e5');
+  assert.equal(press.x, 88);
+  assert.equal(press.y, 99);
+
+  assert.equal(click.targetKind, 'point');
+  assert.equal(click.x, 10);
+  assert.equal(click.y, 20);
+  assert.equal(click.button, 'secondary');
+
+  assert.equal(fill.targetKind, 'ref');
+  assert.equal(fill.ref, 'e5');
+  assert.equal(fill.text, 'hello');
+
+  assert.equal(longPress.targetKind, 'selector');
+  assert.equal(longPress.selector, 'label=Foo');
+  assert.equal(longPress.gesture, 'longpress');
+  assert.equal(longPress.durationMs, 500);
+
+  assert.equal(find.ref, '@e5');
+  assert.equal(find.refsGeneration, 42);
+  assert.equal(find.text, 'Hello');
+
+  assert.deepEqual(
+    [pressType, clickType, fillType, longPressType, findType],
+    [true, true, true, true, true],
+  );
+});
+
+test('interaction responses expose additive cost and direct-iOS Maestro fallback fields', async () => {
+  const setup = createTransport(async (req) => {
+    if (req.command === 'press') {
+      return {
+        ok: true,
+        data: {
+          targetKind: 'ref',
+          ref: 'e5',
+          cost: { wallClockMs: 123, runnerRoundTrips: 2, nodeCount: 5 },
+        },
+      };
+    }
+    if (req.command === 'click') {
+      return {
+        ok: true,
+        data: {
+          targetKind: 'selector',
+          selector: 'id=hidden',
+          maestroNonHittableCoordinateFallbackAllowed: true,
+          maestroNonHittableCoordinateFallbackUsed: true,
+          maestroFallbackReason: 'non-hittable-coordinate',
+        },
+      };
+    }
+    if (req.command === 'find') {
+      return {
+        ok: true,
+        data: {
+          ref: '@e5',
+          refsGeneration: 42,
+          text: 'Hello',
+          cost: { wallClockMs: 45, runnerRoundTrips: 0 },
+        },
+      };
+    }
+    throw new Error(`unexpected command: ${req.command}`);
+  });
+  const client = createAgentDeviceClient(setup.config, { transport: setup.transport });
+
+  const press = await client.interactions.press({ ref: '@e5', cost: true });
+  const click = await client.interactions.click({ selector: 'id=hidden' });
+  const find = await client.interactions.find({
+    locator: 'label',
+    query: 'Foo',
+    action: 'getText',
+    cost: true,
+  });
+
+  assert.equal(press.targetKind, 'ref');
+  assert.equal(press.cost?.wallClockMs, 123);
+  assert.equal(press.cost?.runnerRoundTrips, 2);
+  assert.equal(press.cost?.nodeCount, 5);
+
+  assert.equal(click.targetKind, 'selector');
+  assert.equal(click.selector, 'id=hidden');
+  assert.equal(click.maestroNonHittableCoordinateFallbackAllowed, true);
+  assert.equal(click.maestroNonHittableCoordinateFallbackUsed, true);
+  assert.equal(click.maestroFallbackReason, 'non-hittable-coordinate');
+
+  assert.equal(find.ref, '@e5');
+  assert.equal(find.cost?.wallClockMs, 45);
+  assert.equal(find.cost?.runnerRoundTrips, 0);
+});

src/client/client-types.ts

@@ -480,6 +480,12 @@ export type CaptureSnapshotResult = {
    * token budget); pair a ref with it (`@e12~s<refsGeneration>`) before a mutation.
    */
   refsGeneration?: number;
+  /**
+   * Digest response view only: a capped list of `{ ref, label? }` pairs taken
+   * from the full `nodes` tree so the MCP layer can still pin refs when the
+   * default-level `nodes` payload is intentionally omitted.
+   */
+  refs?: Array<{ ref: string; label?: string }>;
 } & PublicSnapshotCaptureAnnotations;
 
 export type CaptureScreenshotOptions = AgentDeviceRequestOverrides & {
@@ -1190,23 +1196,23 @@ export type AgentDeviceClient = {
     diff: (options: CaptureDiffOptions) => Promise<CommandResult<'diff'>>;
   };
   interactions: {
-    click: (options: ClickOptions) => Promise<CommandRequestResult>;
-    press: (options: PressOptions) => Promise<CommandRequestResult>;
-    longPress: (options: LongPressOptions) => Promise<CommandRequestResult>;
+    click: (options: ClickOptions) => Promise<CommandResult<'click'>>;
+    press: (options: PressOptions) => Promise<CommandResult<'press'>>;
+    longPress: (options: LongPressOptions) => Promise<CommandResult<'longpress'>>;
     swipe: (options: SwipeOptions) => Promise<CommandRequestResult>;
     pan: (options: PanOptions) => Promise<CommandRequestResult>;
     fling: (options: FlingOptions) => Promise<CommandRequestResult>;
     swipeGesture: (options: SwipeGestureOptions) => Promise<CommandRequestResult>;
     focus: (options: FocusOptions) => Promise<CommandRequestResult>;
     type: (options: TypeTextOptions) => Promise<CommandRequestResult>;
-    fill: (options: FillOptions) => Promise<CommandRequestResult>;
+    fill: (options: FillOptions) => Promise<CommandResult<'fill'>>;
     scroll: (options: ScrollOptions) => Promise<CommandRequestResult>;
     pinch: (options: PinchOptions) => Promise<CommandRequestResult>;
     rotateGesture: (options: RotateGestureOptions) => Promise<CommandRequestResult>;
     transformGesture: (options: TransformGestureOptions) => Promise<CommandRequestResult>;
     get: (options: GetOptions) => Promise<CommandRequestResult>;
     is: (options: IsOptions) => Promise<CommandRequestResult>;
-    find: (options: FindOptions) => Promise<CommandRequestResult>;
+    find: (options: FindOptions) => Promise<CommandResult<'find'>>;
   };
   replay: {
     run: (options: ReplayRunOptions) => Promise<CommandResult<'replay'>>;

src/contracts/interaction.ts

@@ -1,4 +1,6 @@
 import type { Point, SnapshotNode } from '../kernel/snapshot.ts';
+import type { ResponseCost } from '../kernel/contracts.ts';
+import type { ClickButton } from '../core/click-button.ts';
 
 export type SelectorTarget = {
   kind: 'selector';
@@ -228,6 +230,93 @@ export type SettleObservation = {
   hint?: string;
 };
 
+/**
+ * Public daemon response data shared by press/click/fill/longpress.
+ * `buildInteractionResponseData` emits this shape (ADR 0011 Layer 2):
+ * `targetKind` discriminates the resolved target, identity fields are FLAT
+ * (`ref`, `selector`, `x`, `y`), and per-command extras ride alongside.
+ */
+type TouchResponseDataBase = {
+  message?: string;
+  warning?: string;
+  x?: number;
+  y?: number;
+  referenceWidth?: number;
+  referenceHeight?: number;
+  evidence?: InteractionEvidence;
+  settle?: SettleObservation;
+  resolution?: ResolutionDisclosure;
+  cost?: ResponseCost;
+  /** Direct iOS Maestro coordinate-fallback signals. */
+  maestroNonHittableCoordinateFallbackAllowed?: boolean;
+  maestroNonHittableCoordinateFallbackUsed?: boolean;
+  maestroFallbackReason?: 'non-hittable-coordinate';
+};
+
+type TouchResponsePoint = TouchResponseDataBase & {
+  targetKind: 'point';
+  x: number;
+  y: number;
+};
+
+type TouchResponseRef = TouchResponseDataBase & {
+  targetKind: 'ref';
+  ref: string;
+  refLabel?: string;
+  selectorChain?: string[];
+  targetHittable?: boolean;
+  hint?: string;
+};
+
+type TouchResponseSelector = TouchResponseDataBase & {
+  targetKind: 'selector';
+  selector: string;
+  selectorChain?: string[];
+  refLabel?: string;
+  targetHittable?: boolean;
+  hint?: string;
+};
+
+type TouchPressExtras = {
+  button?: ClickButton;
+  count?: number;
+  intervalMs?: number;
+  holdMs?: number;
+  jitterPx?: number;
+  doubleTap?: boolean;
+};
+
+export type PressCommandResponseData =
+  | (TouchResponsePoint & TouchPressExtras)
+  | (TouchResponseRef & TouchPressExtras)
+  | (TouchResponseSelector & TouchPressExtras);
+
+export type ClickCommandResponseData = PressCommandResponseData;
+
+type TouchFillExtras = {
+  text: string;
+  delayMs?: number;
+};
+
+export type FillCommandResponseData =
+  | (TouchResponsePoint & TouchFillExtras)
+  | (TouchResponseRef & TouchFillExtras)
+  | (TouchResponseSelector & TouchFillExtras);
+
+type TouchLongPressExtras = {
+  durationMs?: number;
+  gesture: 'longpress';
+};
+
+export type LongPressCommandResponseData =
+  | (TouchResponsePoint & TouchLongPressExtras)
+  | (TouchResponseRef & TouchLongPressExtras)
+  | (TouchResponseSelector & TouchLongPressExtras);
+
+/**
+ * Internal runtime result for press/click. The daemon response layer turns
+ * this into `PressCommandResponseData` via `buildInteractionResponseData`.
+ */
 export type PressCommandResult = ResolvedInteractionTarget & {
   backendResult?: Record<string, unknown>;
   message?: string;
@@ -236,6 +325,10 @@ export type PressCommandResult = ResolvedInteractionTarget & {
   settle?: SettleObservation;
 };
 
+/**
+ * Internal runtime result for fill. The daemon response layer turns this into
+ * `FillCommandResponseData` via `buildInteractionResponseData`.
+ */
 export type FillCommandResult = ResolvedInteractionTarget & {
   text: string;
   warning?: string;
@@ -245,10 +338,38 @@ export type FillCommandResult = ResolvedInteractionTarget & {
   settle?: SettleObservation;
 };
 
+/**
+ * Internal runtime result for longpress. The daemon response layer turns this
+ * into `LongPressCommandResponseData` via `buildInteractionResponseData`.
+ */
 export type LongPressCommandResult = ResolvedInteractionTarget & {
   durationMs?: number;
   backendResult?: Record<string, unknown>;
   message?: string;
   warning?: string;
   settle?: SettleObservation;
 };
+
+/**
+ * Daemon response data for the `find` command. Read-only actions (`exists`,
+ * `wait`, `get_text`, `get_attrs`) may issue a pinnable ref with
+ * `refsGeneration`; mutating actions (`click`, `fill`, `focus`, `type`) carry
+ * `ref` as diagnostic pre-action identity and intentionally omit `refsGeneration`
+ * (ADR 0014). The shape is intentionally a flat, optional-field record because
+ * the action positional changes which fields are present.
+ */
+export type FindCommandResponseData = {
+  ref?: string;
+  refsGeneration?: number;
+  found?: true;
+  waitedMs?: number;
+  text?: string;
+  node?: SnapshotNode;
+  locator?: string;
+  query?: string;
+  x?: number;
+  y?: number;
+  message?: string;
+  settle?: SettleObservation;
+  cost?: ResponseCost;
+};