feat: add agent-friendly simulator CLI shortcuts

DjDeveloperr · DjDeveloperr · commit 078969927a66 · 2026-05-22T21:44:31.000-04:00
diff --git a/README.md b/README.md
@@ -95,6 +95,8 @@ CLI commands automatically use the same warm daemon:
 simdeck list
 simdeck tap <udid> 0.5 0.5 --normalized
 simdeck describe <udid> --format agent --max-depth 2
+SIMDECK_DEVICE=<udid> simdeck tap "Continue"
+simdeck --device <udid> describe --format agent --max-depth 2 --interactive
 ```
 
 ## Daemon
@@ -155,11 +157,13 @@ simdeck record <udid> --seconds 5 --output screen-recording.mp4
 simdeck stream <udid> --frames 120 > stream.h264
 simdeck describe <udid>
 simdeck describe <udid> --format agent --max-depth 4
+simdeck describe <udid> --format agent --max-depth 4 --interactive
 simdeck describe <udid> --point 120,240
 simdeck wait-for <udid> --label "Welcome" --timeout-ms 5000
 simdeck assert <udid> --id login.button --source auto --max-depth 8
 simdeck tap <udid> 120 240
 simdeck tap <udid> --label "Continue" --wait-timeout-ms 5000
+simdeck tap "Continue"
 simdeck swipe <udid> 200 700 200 200
 simdeck gesture <udid> scroll-down
 simdeck pinch <udid> --start-distance 160 --end-distance 80
@@ -214,9 +218,11 @@ external tools such as `ffplay`.
 Flutter, or UIKit in-app inspectors, then falls back to the built-in private
 CoreSimulator accessibility bridge. Use `--format agent` or
 `--format compact-json` for
-lower-token hierarchy dumps. Coordinate commands accept screen coordinates from
-the accessibility tree by default; pass `--normalized` to send `0.0..1.0`
-coordinates directly.
+lower-token hierarchy dumps, and add `--interactive`/`-i` when an agent only
+needs actionable elements plus their ancestors. `describe` and `tap` can infer a
+target from `--device`, `SIMDECK_DEVICE`, `SIMDECK_UDID`, or the only booted
+simulator. Coordinate commands accept screen coordinates from the accessibility
+tree by default; pass `--normalized` to send `0.0..1.0` coordinates directly.
 
 ## JS/TS Tests
 
diff --git a/docs/api/rest.md b/docs/api/rest.md
@@ -207,11 +207,12 @@ Touch, edge-touch, and multi-touch coordinates are normalized from `0.0` to `1.0
 
 Tree query parameters:
 
-| Parameter       | Values                                                                                                    |
-| --------------- | --------------------------------------------------------------------------------------------------------- |
-| `source`        | `auto`, `nativescript`, `react-native`, `flutter`, `swiftui`, `uikit`, `native-ax`, `android-uiautomator` |
-| `maxDepth`      | Integer depth limit                                                                                       |
-| `includeHidden` | `true` or `false`                                                                                         |
+| Parameter         | Values                                                                                                    |
+| ----------------- | --------------------------------------------------------------------------------------------------------- |
+| `source`          | `auto`, `nativescript`, `react-native`, `flutter`, `swiftui`, `uikit`, `native-ax`, `android-uiautomator` |
+| `maxDepth`        | Integer depth limit                                                                                       |
+| `includeHidden`   | `true` or `false`                                                                                         |
+| `interactiveOnly` | `true` keeps actionable elements plus their ancestors                                                     |
 
 Point query parameters:
 
diff --git a/docs/cli/commands.md b/docs/cli/commands.md
@@ -69,6 +69,7 @@ simdeck toggle-appearance <udid>
 ```sh
 simdeck describe <udid>
 simdeck describe <udid> --format agent --max-depth 4
+simdeck describe <udid> --format agent --max-depth 4 --interactive
 simdeck describe <udid> --format compact-json
 simdeck describe <udid> --source nativescript
 simdeck describe <udid> --source react-native
@@ -80,7 +81,7 @@ simdeck wait-for <udid> --label "Welcome" --timeout-ms 5000
 simdeck assert <udid> --id login.button --source auto --max-depth 8
 ```
 
-Default source selection prefers a connected framework inspector, then the Swift in-app agent, then native accessibility.
+Default source selection prefers a connected framework inspector, then the Swift in-app agent, then native accessibility. Use `--interactive` or `-i` to keep actionable elements and the ancestor context needed to find them. For quick agent loops, `describe` can infer the device from `--device`, `SIMDECK_DEVICE`, `SIMDECK_UDID`, or the only booted simulator.
 
 ## Performance
 
@@ -97,12 +98,13 @@ Performance data is simulator-only and uses host-process telemetry for matching
 
 ## Input
 
-Coordinates are screen points unless `--normalized` is present.
+Coordinates are screen points unless `--normalized` is present. `tap "Continue"` is shorthand for a label tap on the inferred device. Use `--device <udid>` or `SIMDECK_DEVICE=<udid>` when more than one simulator is booted.
 
 ```sh
 simdeck tap <udid> 120 240
 simdeck tap <udid> 0.5 0.5 --normalized
 simdeck tap <udid> --label "Continue" --wait-timeout-ms 5000
+simdeck tap "Continue"
 simdeck swipe <udid> 200 700 200 200
 simdeck gesture <udid> scroll-down
 simdeck pinch <udid> --start-distance 160 --end-distance 80
diff --git a/docs/cli/flags.md b/docs/cli/flags.md
@@ -10,36 +10,38 @@ simdeck daemon start --help
 
 ## Global
 
-| Flag                 | Env                  | Purpose                          |
-| -------------------- | -------------------- | -------------------------------- |
-| `--server-url <url>` | `SIMDECK_SERVER_URL` | Target a specific running daemon |
+| Flag                  | Env                  | Purpose                                        |
+| --------------------- | -------------------- | ---------------------------------------------- |
+| `--server-url <url>`  | `SIMDECK_SERVER_URL` | Target a specific running daemon               |
+| `--device <selector>` | `SIMDECK_DEVICE`     | Default simulator for inferred-device commands |
 
 ## Server Options
 
 Used by `simdeck ui`, `daemon start`, `daemon restart`, `service on`, and `service restart`.
 
 | Flag                         | Default        | Notes                                                                               |
-| ---------------------------- | -------------- | ----------------------------------------------------------------------------------- | ------ | ------------ |
+| ---------------------------- | -------------- | ----------------------------------------------------------------------------------- |
 | `--port <port>`              | `4310`         | HTTP port. LaunchAgent service commands probe up to 4320 when this port is occupied |
 | `--bind <ip>`                | `127.0.0.1`    | Use `0.0.0.0` or `::` for LAN access                                                |
 | `--advertise-host <host>`    | detected       | Host printed for remote browsers                                                    |
 | `--client-root <path>`       | bundled client | Static client directory                                                             |
-| `--video-codec auto          | hardware       | software`                                                                           | `auto` | Encoder mode |
+| `--video-codec <mode>`       | `auto`         | `auto`, `hardware`, or `software`                                                   |
 | `--stream-quality <profile>` | `full`         | `full`, `balanced`, `economy`, `low`, `tiny`, `ci-software`, and related profiles   |
 | `--local-stream-fps <fps>`   | `60`           | Local stream frame target                                                           |
 | `--low-latency`              | off            | Conservative software H.264 profile                                                 |
 | `--open`                     | off            | `ui` only                                                                           |
 
 ## `describe`
 
-| Flag               | Purpose                                           |
-| ------------------ | ------------------------------------------------- | ------------ | ------------------- | ----- | --------- | -------------------- | --------------------- |
-| `--format json     | compact-json                                      | agent`       | Choose output shape |
-| `--source auto     | nativescript                                      | react-native | flutter             | uikit | native-ax | android-uiautomator` | Pick inspector source |
-| `--max-depth <n>`  | Trim hierarchy depth                              |
-| `--include-hidden` | Include hidden nodes when supported               |
-| `--point <x>,<y>`  | Describe the element at a screen point            |
-| `--direct`         | Skip daemon and use native accessibility directly |
+| Flag                  | Purpose                                                                                           |
+| --------------------- | ------------------------------------------------------------------------------------------------- |
+| `--format <format>`   | `json`, `compact-json`, or `agent`                                                                |
+| `--source <source>`   | `auto`, `nativescript`, `react-native`, `flutter`, `uikit`, `native-ax`, or `android-uiautomator` |
+| `--max-depth <n>`     | Trim hierarchy depth                                                                              |
+| `--include-hidden`    | Include hidden nodes when supported                                                               |
+| `-i`, `--interactive` | Keep only actionable elements plus ancestors                                                      |
+| `--point <x>,<y>`     | Describe the element at a screen point                                                            |
+| `--direct`            | Skip daemon and use native accessibility directly                                                 |
 
 ## Input
 
diff --git a/docs/cli/index.md b/docs/cli/index.md
@@ -37,7 +37,8 @@ simdeck install <udid> /path/to/App.ipa
 simdeck launch <udid> com.example.App
 simdeck open-url <udid> https://example.com
 simdeck tap <udid> --label "Continue" --wait-timeout-ms 5000
-simdeck describe <udid> --format agent --max-depth 3
+simdeck tap "Continue"
+simdeck describe <udid> --format agent --max-depth 3 --interactive
 simdeck screenshot <udid> --output screen.png
 simdeck screenshot <udid> --with-bezel --output screen-bezel.png
 simdeck record <udid> --seconds 5 --output screen-recording.mp4
@@ -46,6 +47,8 @@ simdeck stats <udid>
 simdeck sample <udid> --seconds 3
 ```
 
+`tap` and `describe` can infer their simulator from `--device`, `SIMDECK_DEVICE`, `SIMDECK_UDID`, or the only booted simulator.
+
 Most successful commands print JSON so they can be piped into tools such as `jq`.
 
 ## Help
diff --git a/docs/guide/index.md b/docs/guide/index.md
@@ -30,7 +30,8 @@ simdeck install <udid> /path/to/App.app
 simdeck install <udid> /path/to/App.ipa
 simdeck launch <udid> com.example.App
 simdeck tap <udid> --label "Continue" --wait-timeout-ms 5000
-simdeck describe <udid> --format agent --max-depth 3
+simdeck tap "Continue"
+simdeck describe <udid> --format agent --max-depth 3 --interactive
 ```
 
 Use `simdeck -d` for a detached background daemon, `simdeck -k` to stop it, and `simdeck -r` to restart it.
diff --git a/docs/guide/quick-start.md b/docs/guide/quick-start.md
@@ -66,7 +66,8 @@ Use selectors when you want automation to wait for UI state:
 
 ```sh
 simdeck tap <udid> --label "Continue" --wait-timeout-ms 5000
-simdeck describe <udid> --format agent --max-depth 3
+simdeck tap "Continue"
+simdeck describe <udid> --format agent --max-depth 3 --interactive
 ```
 
 ## 5. Keep It Running In The Background
diff --git a/docs/inspector/index.md b/docs/inspector/index.md
@@ -20,11 +20,12 @@ Use the built-in accessibility fallback for any app. Add an in-app inspector whe
 ```sh
 simdeck describe <udid>
 simdeck describe <udid> --format agent --max-depth 3
+simdeck describe <udid> --format agent --max-depth 3 --interactive
 simdeck describe <udid> --source native-ax
 simdeck describe <udid> --source react-native
 ```
 
-`auto` source selection uses the best available source and falls back to accessibility.
+`auto` source selection uses the best available source and falls back to accessibility. Add `--interactive` or `-i` for a smaller agent-oriented tree of actionable elements plus ancestors.
 
 ## Use From The Browser
 
diff --git a/packages/simdeck-test/dist/index.d.ts b/packages/simdeck-test/dist/index.d.ts
@@ -11,6 +11,7 @@ export type QueryOptions = {
     source?: "auto" | "nativescript" | "react-native" | "flutter" | "swiftui" | "uikit" | "native-ax" | "android-uiautomator";
     maxDepth?: number;
     includeHidden?: boolean;
+    interactiveOnly?: boolean;
 };
 export type ElementSelector = {
     text?: string;
diff --git a/packages/simdeck-test/dist/index.js b/packages/simdeck-test/dist/index.js
@@ -524,6 +524,8 @@ function treeQuery(options = {}) {
         params.set("maxDepth", String(options.maxDepth));
     if (options.includeHidden)
         params.set("includeHidden", "true");
+    if (options.interactiveOnly)
+        params.set("interactiveOnly", "true");
     return params.toString();
 }
 function logsQuery(options = {}) {
diff --git a/packages/simdeck-test/src/index.ts b/packages/simdeck-test/src/index.ts
@@ -28,6 +28,7 @@ export type QueryOptions = {
     | "android-uiautomator";
   maxDepth?: number;
   includeHidden?: boolean;
+  interactiveOnly?: boolean;
 };
 
 export type ElementSelector = {
@@ -923,6 +924,7 @@ function treeQuery(options: QueryOptions = {}): string {
   if (options.maxDepth !== undefined)
     params.set("maxDepth", String(options.maxDepth));
   if (options.includeHidden) params.set("includeHidden", "true");
+  if (options.interactiveOnly) params.set("interactiveOnly", "true");
   return params.toString();
 }
 
diff --git a/scripts/integration/cli.mjs b/scripts/integration/cli.mjs
@@ -189,6 +189,29 @@ async function main() {
   if (!agentTree.includes("source:") || !agentTree.includes("- ")) {
     throw new Error("agent describe output did not look like a hierarchy");
   }
+  const interactiveTree = await measuredStep(
+    "server describe agent interactive",
+    () =>
+      simdeckText([
+        "describe",
+        simulatorUDID,
+        "--source",
+        "native-ax",
+        "--format",
+        "agent",
+        "--max-depth",
+        "8",
+        "--interactive",
+      ]),
+  );
+  if (
+    !interactiveTree.includes("source:") ||
+    !interactiveTree.includes("Continue")
+  ) {
+    throw new Error(
+      "interactive agent describe did not include fixture controls",
+    );
+  }
   await runRestControls();
   await runCliControls();
 
@@ -482,6 +505,16 @@ async function runCliControls() {
     },
     { expectFixture: true, expectText: "URL Opened" },
   );
+  await cliStep(
+    "CLI tap label shorthand",
+    ["tap", "Continue", "--wait-timeout-ms", "15000", "--duration-ms", "30"],
+    {
+      env: { SIMDECK_DEVICE: simulatorUDID },
+      timeoutMs: 180_000,
+      maxElapsedMs: 60_000,
+    },
+    { expectFixture: true, expectText: "Continue Tapped", attempts: 8 },
+  );
   await cliStep(
     "CLI tap fixture text field",
     [
@@ -497,7 +530,7 @@ async function runCliControls() {
     { timeoutMs: 180_000, maxElapsedMs: 60_000 },
     {
       expectFixture: true,
-      expectText: "URL Opened",
+      expectText: "Continue Tapped",
       attempts: 6,
       delayMs: 1_500,
     },
@@ -623,6 +656,20 @@ async function runRestControls() {
     },
     { phase: phaseSetup },
   );
+  await measuredStep(
+    "REST accessibility-tree interactive",
+    async () => {
+      const tree = await httpJson(
+        "GET",
+        `/api/simulators/${simulatorUDID}/accessibility-tree?source=native-ax&maxDepth=8&interactiveOnly=true`,
+      );
+      assertRoots(tree, "REST accessibility-tree interactive");
+      if (tree.interactiveOnly !== true) {
+        throw new Error("interactive tree did not report interactiveOnly=true");
+      }
+    },
+    { phase: phaseSetup },
+  );
   await measuredStep(
     "REST chrome-profile",
     async () => {
@@ -1640,6 +1687,7 @@ function simdeckText(args, options = {}) {
     timeoutMs: options.timeoutMs ?? 120_000,
     maxElapsedMs: options.maxElapsedMs,
     input: options.input,
+    env: options.env,
   });
 }
 
@@ -1654,6 +1702,7 @@ function runText(command, args, options = {}) {
     cwd: root,
     encoding: "utf8",
     input: options.input,
+    env: options.env ? { ...process.env, ...options.env } : process.env,
     timeout: options.timeoutMs ?? 120_000,
   });
   if (result.status !== 0) {
diff --git a/server/src/accessibility.rs b/server/src/accessibility.rs
diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs
diff --git a/server/src/main.rs b/server/src/main.rs
diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md

Original file line number	Diff line number	Diff line change
`@@ -524,6 +524,8 @@ function treeQuery(options = {}) {`
`524`	`524`	`params.set("maxDepth", String(options.maxDepth));`
`525`	`525`	`if (options.includeHidden)`
`526`	`526`	`params.set("includeHidden", "true");`
	`527`	`+ if (options.interactiveOnly)`
	`528`	`+ params.set("interactiveOnly", "true");`
`527`	`529`	`return params.toString();`
`528`	`530`	`}`
`529`	`531`	`function logsQuery(options = {}) {`