feat(simdrive): a11y journeys + set_text for alert fields (1.0.0b10)

TestKitApp/Sources/FormTab.swift

@@ -7,10 +7,21 @@ struct FormTab: View {
     @State private var searchText = ""
     @State private var notes = ""
     @State private var resultText = "Fill the form and tap Submit"
+    // "Go to Page"-style UIAlertController prompt — exercises text entry into an
+    // alert field (which HID keystrokes can't reach; host-AX set-value can).
+    @State private var showGoToPage = false
+    @State private var pageInput = ""
+    @State private var goResult = "no page yet"
 
     var body: some View {
         NavigationView {
             Form {
+                Section("Go to Page") {
+                    Button("Go to Page…") { showGoToPage = true }
+                        .accessibilityIdentifier("btn_go_to_page")
+                    Text(goResult)
+                        .accessibilityIdentifier("lbl_go_result")
+                }
                 Section("Identity") {
                     TextField("First Name", text: $firstName)
                         .accessibilityIdentifier("field_first_name")
@@ -47,6 +58,12 @@ struct FormTab: View {
                 }
             }
             .navigationTitle("TestKit")
+            .alert("Go to Page", isPresented: $showGoToPage) {
+                TextField("Enter a page number", text: $pageInput)
+                    .accessibilityIdentifier("field_go_to_page")
+                Button("Go") { goResult = "Went to page \(pageInput)" }
+                Button("Cancel", role: .cancel) { }
+            }
         }
     }
 }

simdrive/CHANGELOG.md

@@ -13,6 +13,20 @@
  filter catch what slips through.
 -->
 
+## [1.0.0b10] — 2026-06-22
+
+### Added — text entry into alert fields + accessibility journeys
+
+- **`set_text`** — set a text field's value directly via the host Accessibility
+  API, for fields that `type_text` (synthesized keystrokes) can't reach — most
+  importantly **`UIAlertController` prompts** (e.g. a "Go to Page" dialog). The
+  value propagates to the field's binding, so the app receives it. Targets the
+  field by `identifier`/`label`, or defaults to the alert's field. Simulator-only.
+- **Journeys** can now use the accessibility tools: a `perform_accessibility_action`
+  step and an **`announcement_heard`** success criterion, so an end-to-end
+  VoiceOver flow (drive → invoke a custom action → assert the spoken announcement)
+  is expressible as a journey.
+
 ## [1.0.0b9] — 2026-06-22
 
 ### Added — accessibility automation (custom actions + VoiceOver announcements)

simdrive/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "simdrive"
-version = "1.0.0b9"
+version = "1.0.0b10"
 description = "SimDrive — reproduce and validate iOS bugs in 60 seconds with Claude. MCP-native iOS automation; 32 vision-first tools; record once, replay free in CI. 14-day trial, then $29/mo Pro."
 readme = "README.md"
 license = "Elastic-2.0"

simdrive/src/simdrive/ax.py

@@ -334,6 +334,60 @@ def perform_action(
     return {"ok": True, "action": name}
 
 
+_EDITABLE_ROLES = {"AXTextField", "AXSecureTextField", "AXTextArea"}
+
+
+def _find_text_field(root, depth=0, maxdepth=60):
+    if depth > maxdepth:
+        return None
+    if str(_attr(root, "AXRole") or "") in _EDITABLE_ROLES:
+        return root
+    for child in _children(root):
+        hit = _find_text_field(child, depth + 1, maxdepth)
+        if hit is not None:
+            return hit
+    return None
+
+
+def set_text(
+    device_name: str,
+    text: str,
+    *,
+    identifier: str | None = None,
+    label: str | None = None,
+) -> dict[str, Any]:
+    """Set a text field's value directly via host AX (`AXValue`).
+
+    The fix for fields HID `type_text` can't reach — notably `UIAlertController`
+    prompts (e.g. a "Go to Page" dialog), whose field never receives synthesized
+    keystrokes. Setting `AXValue` propagates to the field's binding (verified:
+    the app reads the value), so the app receives the input.
+
+    Resolution: scope to the device's window, then the field by *identifier* /
+    *label*, else the first editable field (text field / secure field / text
+    area) in the window — which is the alert's field when a prompt is up.
+
+    Returns ``{"ok": True, "value": text}`` or ``{"ok": False, "error": ...}``.
+    """
+    from ApplicationServices import AXUIElementSetAttributeValue  # type: ignore[import]
+
+    window = select_window(device_name)
+    if identifier:
+        field = _find_by(window, "AXIdentifier", identifier)
+    elif label:
+        field = _find_by(window, "AXDescription", label) or _find_by(window, "AXTitle", label)
+    else:
+        field = _find_text_field(window)
+
+    if field is None:
+        return {"ok": False, "error": "no editable text field found in the target window"}
+
+    err = AXUIElementSetAttributeValue(field, "AXValue", text)
+    if err != 0:
+        return {"ok": False, "error": f"AXUIElementSetAttributeValue err={err}"}
+    return {"ok": True, "value": text}
+
+
 # ---------------------------------------------------------------------------
 # Announcement observer (module singleton — one Simulator process)
 # ---------------------------------------------------------------------------

simdrive/src/simdrive/journey/criteria.py

@@ -47,6 +47,28 @@ def eval_text_visible(criterion: SuccessCriterion, obs: dict) -> CriterionEval:
     )
 
 
+def eval_announcement_heard(criterion: SuccessCriterion, obs: dict) -> CriterionEval:
+    """Check that criterion.announcement_heard was posted as a VoiceOver announcement.
+
+    The runner injects the announcements captured so far into ``obs["announcements"]``
+    (list of ``{text, ...}``); matching is case-insensitive substring.
+    """
+    target = criterion.announcement_heard
+    if target is None:
+        raise ValueError("eval_announcement_heard called but criterion.announcement_heard is None")
+
+    announcements = obs.get("announcements", [])
+    heard = [a.get("text", "") for a in announcements]
+    found = any(target.lower() in t.lower() for t in heard)
+
+    return CriterionEval(
+        criterion_type="announcement_heard",
+        passed=found,
+        detail=f"announcement {target!r} {'heard' if found else 'not heard'}",
+        observed_value=heard[-5:] if heard else [],
+    )
+
+
 def eval_screen_matches(criterion: SuccessCriterion, obs: dict) -> CriterionEval:
     """Check that criterion.screen_matches stable_id appears in the current marks."""
     stable_id = criterion.screen_matches
@@ -191,6 +213,8 @@ def evaluate_all_criteria(
                 results.append(eval_no_crash(criterion, crashes))
             elif criterion.cross_device_state_matches is not None:
                 results.append(eval_cross_device_state_matches(criterion))
+            elif criterion.announcement_heard is not None:
+                results.append(eval_announcement_heard(criterion, obs))
             else:
                 # Should never happen — SuccessCriterion validates at least one field.
                 results.append(

simdrive/src/simdrive/journey/prompt.py

@@ -46,14 +46,24 @@
 - The list of success criteria still unmet
 
 You must respond with a JSON object with these fields:
-  tool: one of "tap" | "swipe" | "type_text" | "press_key" | "clear_field" | "done" | "fail"
+  tool: one of "tap" | "swipe" | "type_text" | "press_key" | "clear_field" | "perform_accessibility_action" | "set_text" | "done" | "fail"
   args: a dict of arguments for the chosen tool (empty dict for "done"/"fail")
   rationale: one sentence explaining WHY you chose this action
   confidence: float 0.0-1.0 for how confident you are this action advances the goal
 
 Use "done" when ALL success criteria are met.
 Use "fail" only when the app is in a clearly broken state and no recovery is possible.
 
+Use "perform_accessibility_action" (simulator only) to invoke a VoiceOver custom
+action — args {{"name": "<action label>"}}, e.g. {{"name": "Where am I?"}}. These
+are invisible to the screenshot; use them when a goal/criterion refers to a
+VoiceOver rotor action or a spoken announcement.
+
+Use "set_text" (simulator only) to enter text into a field that "type_text"
+can't reach — notably a UIAlertController prompt (e.g. "Go to Page"). args
+{{"text": "..."}} (add {{"identifier": "..."}} or {{"label": "..."}} to target a
+specific field; otherwise the alert's field is used).
+
 Prefer stable_id over pixel coordinates when stable_ids are visible in the marks.
 Be precise with type_text — only type text explicitly required by the goal.
 Never guess or hallucinate element positions.

simdrive/src/simdrive/journey/runner.py

@@ -83,6 +83,21 @@ def tool_crashes(arguments: dict) -> dict:  # pragma: no cover
     from simdrive.server import tool_crashes as _fn
     return _fn(arguments)
 
+
+def tool_perform_accessibility_action(arguments: dict) -> dict:  # pragma: no cover
+    from simdrive.server import tool_perform_accessibility_action as _fn
+    return _fn(arguments)
+
+
+def tool_get_announcements(arguments: dict) -> dict:  # pragma: no cover
+    from simdrive.server import tool_get_announcements as _fn
+    return _fn(arguments)
+
+
+def tool_set_text(arguments: dict) -> dict:  # pragma: no cover
+    from simdrive.server import tool_set_text as _fn
+    return _fn(arguments)
+
 # Approximate cost per LLM call (vision, claude-3-7 range); used for
 # in-flight cost tracking when the real client doesn't report token counts.
 _APPROX_COST_PER_CALL_USD = 0.004
@@ -98,7 +113,10 @@ def tool_crashes(arguments: dict) -> dict:  # pragma: no cover
 class StepDecision:
     """Parsed response from one LLM vision call."""
 
-    tool: Literal["tap", "swipe", "type_text", "press_key", "clear_field", "done", "fail"]
+    tool: Literal[
+        "tap", "swipe", "type_text", "press_key", "clear_field",
+        "perform_accessibility_action", "set_text", "done", "fail",
+    ]
     args: dict
     rationale: str
     confidence: float
@@ -205,6 +223,8 @@ def _dispatch_action(decision: StepDecision, session_id: str) -> None:
         "type_text": _self.tool_type_text,
         "press_key": _self.tool_press_key,
         "clear_field": _self.tool_clear_field,
+        "perform_accessibility_action": _self.tool_perform_accessibility_action,
+        "set_text": _self.tool_set_text,
     }
 
     tool_fn = tool_map.get(decision.tool)
@@ -352,6 +372,14 @@ async def run_journey(
             failure_reason = f"observe failed: {exc}"
             break
 
+        # Inject VoiceOver announcements captured so far (simulator-only) so an
+        # announcement_heard criterion can be evaluated. Best-effort.
+        try:
+            ann = tool_get_announcements({"session_id": session_id})
+            obs_dict["announcements"] = ann.get("announcements", [])
+        except Exception:
+            obs_dict.setdefault("announcements", [])
+
         # Evaluate success criteria against current observation.
         perf_snapshot: Optional[dict] = None
         try:

simdrive/src/simdrive/journey/schema.py

@@ -34,6 +34,9 @@ class SuccessCriterion(BaseModel):
     no_crash: Optional[bool] = None
     # 1.0 stretch — pass-through when not configured (never fail-closed)
     cross_device_state_matches: Optional[dict[str, Any]] = None
+    # Host-AX: a VoiceOver announcement (substring, case-insensitive) the app
+    # must have posted during the journey (simulator sessions only).
+    announcement_heard: Optional[str] = None
 
     @model_validator(mode="after")
     def _at_least_one_criterion(self) -> "SuccessCriterion":
@@ -46,11 +49,13 @@ def _at_least_one_criterion(self) -> "SuccessCriterion":
                 self.perf_under,
                 self.no_crash,
                 self.cross_device_state_matches,
+                self.announcement_heard,
             )
         ):
             raise ValueError(
-                "SuccessCriterion must set at least one field "
-                "(text_visible, screen_matches, perf_under, no_crash, cross_device_state_matches)"
+                "SuccessCriterion must set at least one field (text_visible, "
+                "screen_matches, perf_under, no_crash, cross_device_state_matches, "
+                "announcement_heard)"
             )
         return self
 

simdrive/src/simdrive/server.py

@@ -1982,6 +1982,35 @@ def tool_perform_accessibility_action(arguments: dict) -> dict:
     return result
 
 
+def tool_set_text(arguments: dict) -> dict:
+    """Set a text field's value directly via host AX (sim-only).
+
+    For fields HID `type_text` can't reach — e.g. `UIAlertController` prompts.
+    """
+    _entitlement_gate()
+    s = session.get(arguments["session_id"])
+    if s.target != "simulator":
+        raise errors.invalid_argument(
+            "target", s.target, "set_text is simulator-only (host AX); use type_text on device"
+        )
+    text = arguments.get("text")
+    if text is None:
+        raise errors.invalid_argument("text", text, "text is required")
+    try:
+        result = ax.set_text(
+            s.device.name,
+            str(text),
+            identifier=arguments.get("identifier"),
+            label=arguments.get("label"),
+        )
+    except ax.AXError as exc:
+        raise errors.invalid_argument("session_id", arguments["session_id"], str(exc)) from exc
+    s.last_action_at = _now()
+    session.append_action(s, {"action": "set_text", "args": dict(arguments),
+                              "result": result, "at": _now()})
+    return result
+
+
 def tool_get_announcements(arguments: dict) -> dict:
     """Return VoiceOver announcements captured by the host AX observer (sim-only)."""
     _entitlement_gate()
@@ -2368,6 +2397,28 @@ def tool_load_journey(arguments: dict) -> dict:
         },
         "handler": tool_perform_accessibility_action,
     },
+    {
+        "name": "set_text",
+        "description": (
+            "(sim only) Set a text field's value directly via the host Accessibility "
+            "API — for fields that HID `type_text` can't reach, notably "
+            "UIAlertController prompts (e.g. a 'Go to Page' dialog). Targets the field "
+            "by `identifier`/`label`, else the first editable field in the window "
+            "(which is the alert's field when a prompt is up). The value propagates to "
+            "the field's binding, so the app receives it. Returns {ok, value}."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "required": ["session_id", "text"],
+            "properties": {
+                "session_id": {"type": "string"},
+                "text": {"type": "string", "description": "Text to set into the field."},
+                "identifier": {"type": "string", "description": "Optional accessibilityIdentifier of the target field."},
+                "label": {"type": "string", "description": "Optional accessibility label/description of the target field."},
+            },
+        },
+        "handler": tool_set_text,
+    },
     {
         "name": "get_announcements",
         "description": (

simdrive/tests/test_ax_module.py

@@ -39,6 +39,21 @@ def __init__(self, actions=None, children=None):
         self.children = children or []
 
 
+def test_find_text_field_recurses(monkeypatch):
+    class FE:
+        def __init__(self, role="", children=None):
+            self.role = role
+            self.children = children or []
+
+    monkeypatch.setattr(ax, "_attr", lambda e, a: e.role if a == "AXRole" else None)
+    monkeypatch.setattr(ax, "_children", lambda e: e.children)
+
+    field = FE(role="AXTextField")
+    root = FE(children=[FE(role="AXButton"), FE(children=[field])])
+    assert ax._find_text_field(root) is field
+    assert ax._find_text_field(FE(children=[FE(role="AXButton")])) is None
+
+
 def test_find_action_carrier_recurses_to_deep_child(monkeypatch):
     monkeypatch.setattr(ax, "_action_names", lambda e: e.actions)
     monkeypatch.setattr(ax, "_children", lambda e: e.children)
@@ -116,7 +131,7 @@ def test_ax_tools_registered():
     from simdrive import server
 
     names = {t["name"] for t in server._TOOLS}
-    assert {"perform_accessibility_action", "get_announcements"} <= names
+    assert {"perform_accessibility_action", "get_announcements", "set_text"} <= names
     # handlers are callable and listed without the handler key in list_tools()
     listed = {t["name"] for t in server.list_tools()}
-    assert {"perform_accessibility_action", "get_announcements"} <= listed
+    assert {"perform_accessibility_action", "get_announcements", "set_text"} <= listed