From 2616a283db22ae9351760dc0e4846b76f6449ca0 Mon Sep 17 00:00:00 2001 From: Maurice Carrier Date: Mon, 22 Jun 2026 11:35:47 -0400 Subject: [PATCH 1/2] feat(simdrive): journey support for accessibility actions + announcements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the host-AX tools into the LLM journey runner: - New step tool `perform_accessibility_action` — added to StepDecision's tool set, the runner dispatch map, and the system prompt (so the agent can invoke a VoiceOver custom action by name during a journey). - New success criterion `announcement_heard: ""` — the runner injects the announcements captured so far into the observation (simulator-only, best-effort), and a pure evaluator checks the spoken text (case-insensitive substring). This makes a full accessibility journey assertable end-to-end: drive to the reader → perform "Where am I?" → succeed on announcement_heard. Tests: tests/test_journey_ax.py (7); full non-live suite green (1580 passed). **Scope:** journey step + criterion + prompt + tests. **Deferred:** UIAlertController text entry (next). Co-Authored-By: Claude Opus 4.8 (1M context) --- simdrive/src/simdrive/journey/criteria.py | 24 +++++++ simdrive/src/simdrive/journey/prompt.py | 7 +- simdrive/src/simdrive/journey/runner.py | 24 ++++++- simdrive/src/simdrive/journey/schema.py | 9 ++- simdrive/tests/test_journey_ax.py | 78 +++++++++++++++++++++++ 5 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 simdrive/tests/test_journey_ax.py diff --git a/simdrive/src/simdrive/journey/criteria.py b/simdrive/src/simdrive/journey/criteria.py index 9491590f..1a3e5b3f 100644 --- a/simdrive/src/simdrive/journey/criteria.py +++ b/simdrive/src/simdrive/journey/criteria.py @@ -47,6 +47,28 @@ def eval_text_visible(criterion: SuccessCriterion, obs: dict) -> CriterionEval: ) +def eval_announcement_heard(criterion: SuccessCriterion, obs: dict) -> CriterionEval: + """Check that criterion.announcement_heard was posted as a VoiceOver announcement. + + The runner injects the announcements captured so far into ``obs["announcements"]`` + (list of ``{text, ...}``); matching is case-insensitive substring. + """ + target = criterion.announcement_heard + if target is None: + raise ValueError("eval_announcement_heard called but criterion.announcement_heard is None") + + announcements = obs.get("announcements", []) + heard = [a.get("text", "") for a in announcements] + found = any(target.lower() in t.lower() for t in heard) + + return CriterionEval( + criterion_type="announcement_heard", + passed=found, + detail=f"announcement {target!r} {'heard' if found else 'not heard'}", + observed_value=heard[-5:] if heard else [], + ) + + def eval_screen_matches(criterion: SuccessCriterion, obs: dict) -> CriterionEval: """Check that criterion.screen_matches stable_id appears in the current marks.""" stable_id = criterion.screen_matches @@ -191,6 +213,8 @@ def evaluate_all_criteria( results.append(eval_no_crash(criterion, crashes)) elif criterion.cross_device_state_matches is not None: results.append(eval_cross_device_state_matches(criterion)) + elif criterion.announcement_heard is not None: + results.append(eval_announcement_heard(criterion, obs)) else: # Should never happen — SuccessCriterion validates at least one field. results.append( diff --git a/simdrive/src/simdrive/journey/prompt.py b/simdrive/src/simdrive/journey/prompt.py index 559c422e..6a4ca225 100644 --- a/simdrive/src/simdrive/journey/prompt.py +++ b/simdrive/src/simdrive/journey/prompt.py @@ -46,7 +46,7 @@ - The list of success criteria still unmet You must respond with a JSON object with these fields: - tool: one of "tap" | "swipe" | "type_text" | "press_key" | "clear_field" | "done" | "fail" + tool: one of "tap" | "swipe" | "type_text" | "press_key" | "clear_field" | "perform_accessibility_action" | "done" | "fail" args: a dict of arguments for the chosen tool (empty dict for "done"/"fail") rationale: one sentence explaining WHY you chose this action confidence: float 0.0-1.0 for how confident you are this action advances the goal @@ -54,6 +54,11 @@ Use "done" when ALL success criteria are met. Use "fail" only when the app is in a clearly broken state and no recovery is possible. +Use "perform_accessibility_action" (simulator only) to invoke a VoiceOver custom +action — args {{"name": ""}}, e.g. {{"name": "Where am I?"}}. These +are invisible to the screenshot; use them when a goal/criterion refers to a +VoiceOver rotor action or a spoken announcement. + Prefer stable_id over pixel coordinates when stable_ids are visible in the marks. Be precise with type_text — only type text explicitly required by the goal. Never guess or hallucinate element positions. diff --git a/simdrive/src/simdrive/journey/runner.py b/simdrive/src/simdrive/journey/runner.py index a14d8c05..c38e6fe6 100644 --- a/simdrive/src/simdrive/journey/runner.py +++ b/simdrive/src/simdrive/journey/runner.py @@ -83,6 +83,16 @@ def tool_crashes(arguments: dict) -> dict: # pragma: no cover from simdrive.server import tool_crashes as _fn return _fn(arguments) + +def tool_perform_accessibility_action(arguments: dict) -> dict: # pragma: no cover + from simdrive.server import tool_perform_accessibility_action as _fn + return _fn(arguments) + + +def tool_get_announcements(arguments: dict) -> dict: # pragma: no cover + from simdrive.server import tool_get_announcements as _fn + return _fn(arguments) + # Approximate cost per LLM call (vision, claude-3-7 range); used for # in-flight cost tracking when the real client doesn't report token counts. _APPROX_COST_PER_CALL_USD = 0.004 @@ -98,7 +108,10 @@ def tool_crashes(arguments: dict) -> dict: # pragma: no cover class StepDecision: """Parsed response from one LLM vision call.""" - tool: Literal["tap", "swipe", "type_text", "press_key", "clear_field", "done", "fail"] + tool: Literal[ + "tap", "swipe", "type_text", "press_key", "clear_field", + "perform_accessibility_action", "done", "fail", + ] args: dict rationale: str confidence: float @@ -205,6 +218,7 @@ def _dispatch_action(decision: StepDecision, session_id: str) -> None: "type_text": _self.tool_type_text, "press_key": _self.tool_press_key, "clear_field": _self.tool_clear_field, + "perform_accessibility_action": _self.tool_perform_accessibility_action, } tool_fn = tool_map.get(decision.tool) @@ -352,6 +366,14 @@ async def run_journey( failure_reason = f"observe failed: {exc}" break + # Inject VoiceOver announcements captured so far (simulator-only) so an + # announcement_heard criterion can be evaluated. Best-effort. + try: + ann = tool_get_announcements({"session_id": session_id}) + obs_dict["announcements"] = ann.get("announcements", []) + except Exception: + obs_dict.setdefault("announcements", []) + # Evaluate success criteria against current observation. perf_snapshot: Optional[dict] = None try: diff --git a/simdrive/src/simdrive/journey/schema.py b/simdrive/src/simdrive/journey/schema.py index 2f8e8807..3ac6dcbc 100644 --- a/simdrive/src/simdrive/journey/schema.py +++ b/simdrive/src/simdrive/journey/schema.py @@ -34,6 +34,9 @@ class SuccessCriterion(BaseModel): no_crash: Optional[bool] = None # 1.0 stretch — pass-through when not configured (never fail-closed) cross_device_state_matches: Optional[dict[str, Any]] = None + # Host-AX: a VoiceOver announcement (substring, case-insensitive) the app + # must have posted during the journey (simulator sessions only). + announcement_heard: Optional[str] = None @model_validator(mode="after") def _at_least_one_criterion(self) -> "SuccessCriterion": @@ -46,11 +49,13 @@ def _at_least_one_criterion(self) -> "SuccessCriterion": self.perf_under, self.no_crash, self.cross_device_state_matches, + self.announcement_heard, ) ): raise ValueError( - "SuccessCriterion must set at least one field " - "(text_visible, screen_matches, perf_under, no_crash, cross_device_state_matches)" + "SuccessCriterion must set at least one field (text_visible, " + "screen_matches, perf_under, no_crash, cross_device_state_matches, " + "announcement_heard)" ) return self diff --git a/simdrive/tests/test_journey_ax.py b/simdrive/tests/test_journey_ax.py new file mode 100644 index 00000000..c42ad40c --- /dev/null +++ b/simdrive/tests/test_journey_ax.py @@ -0,0 +1,78 @@ +"""Journey integration for host-AX: perform_accessibility_action step + +announcement_heard success criterion. Pure-function / patched-tool tests.""" +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from simdrive.journey.criteria import eval_announcement_heard, evaluate_all_criteria +from simdrive.journey.runner import StepDecision, _dispatch_action +from simdrive.journey.schema import SuccessCriterion + + +# ── announcement_heard criterion ───────────────────────────────────────────── + + +def _obs_with_announcements(*texts: str) -> dict: + return {"marks": [], "announcements": [{"text": t} for t in texts]} + + +def test_announcement_heard_found_substring(): + crit = SuccessCriterion(announcement_heard="page 12") + ev = eval_announcement_heard(crit, _obs_with_announcements("Section 2, page 12, 34%")) + assert ev.passed is True + assert ev.criterion_type == "announcement_heard" + + +def test_announcement_heard_not_found(): + crit = SuccessCriterion(announcement_heard="page 99") + ev = eval_announcement_heard(crit, _obs_with_announcements("Section 1, page 1, 5%")) + assert ev.passed is False + + +def test_announcement_heard_empty_obs(): + crit = SuccessCriterion(announcement_heard="anything") + ev = eval_announcement_heard(crit, {"marks": []}) # no announcements key + assert ev.passed is False + + +def test_evaluate_all_criteria_dispatches_announcement(): + crits = [SuccessCriterion(announcement_heard="Page 2")] + evals = evaluate_all_criteria(crits, obs=_obs_with_announcements("Page 2")) + assert len(evals) == 1 + assert evals[0].criterion_type == "announcement_heard" + assert evals[0].passed is True + + +def test_announcement_heard_is_a_valid_sole_criterion(): + # Must not trip the "at least one field" validator. + crit = SuccessCriterion(announcement_heard="x") + assert crit.announcement_heard == "x" + + +# ── perform_accessibility_action step dispatch ─────────────────────────────── + + +def test_dispatch_routes_perform_accessibility_action(): + decision = StepDecision( + tool="perform_accessibility_action", + args={"name": "Where am I?"}, + rationale="assert position", + confidence=0.9, + ) + with patch( + "simdrive.journey.runner.tool_perform_accessibility_action", + return_value={"ok": True}, + ) as fn: + _dispatch_action(decision, "sess-1") + fn.assert_called_once() + passed = fn.call_args[0][0] + assert passed["name"] == "Where am I?" + assert passed["session_id"] == "sess-1" + + +def test_step_decision_accepts_new_tool_literal(): + # Constructing with the new tool name must not raise. + d = StepDecision(tool="perform_accessibility_action", args={}, rationale="r", confidence=0.5) + assert d.tool == "perform_accessibility_action" From 6436f992c97dfb651c1732b283f9b8ffe4d7fc17 Mon Sep 17 00:00:00 2001 From: Maurice Carrier Date: Mon, 22 Jun 2026 11:45:17 -0400 Subject: [PATCH 2/2] feat(simdrive): set_text for alert fields via host-AX (1.0.0b10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HID type_text can't land in a UIAlertController prompt (e.g. a "Go to Page" dialog) — the field never receives synthesized keystrokes. Add a host-AX set-value path that the app actually receives: - ax.set_text() + tool `set_text`: set a field's AXValue directly. Verified the value propagates to the field's binding (the app reads it), against a real SwiftUI .alert text field. Targets by identifier/label, else the first editable field in the window (the alert's field when a prompt is up). - Journey wiring: `set_text` added to the runner's tool set + dispatch + prompt (alongside the perform_accessibility_action step from the previous commit). - TestKitApp FormTab gains a "Go to Page" alert fixture to exercise it. Tool count 35 → 36; bumps 1.0.0b9 → b10. Tests: ax set_text field-find + tool registration + journey dispatch. Full non-live suite green (1583 passed). **Scope:** set_text + alert fixture + journey wiring + tests. **Deferred:** label-scoping by placeholder text (AXPlaceholderValue) — first-field default covers alerts; device-target alert entry (WDA) unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- TestKitApp/Sources/FormTab.swift | 17 ++++++++ simdrive/CHANGELOG.md | 14 +++++++ simdrive/pyproject.toml | 2 +- simdrive/src/simdrive/ax.py | 54 +++++++++++++++++++++++++ simdrive/src/simdrive/journey/prompt.py | 7 +++- simdrive/src/simdrive/journey/runner.py | 8 +++- simdrive/src/simdrive/server.py | 51 +++++++++++++++++++++++ simdrive/tests/test_ax_module.py | 19 ++++++++- simdrive/tests/test_journey_ax.py | 16 +++++++- simdrive/tests/test_paywall_gates.py | 3 +- simdrive/tests/test_unit.py | 4 +- 11 files changed, 187 insertions(+), 8 deletions(-) diff --git a/TestKitApp/Sources/FormTab.swift b/TestKitApp/Sources/FormTab.swift index 3dd5bc7e..cbddbae2 100644 --- a/TestKitApp/Sources/FormTab.swift +++ b/TestKitApp/Sources/FormTab.swift @@ -7,10 +7,21 @@ struct FormTab: View { @State private var searchText = "" @State private var notes = "" @State private var resultText = "Fill the form and tap Submit" + // "Go to Page"-style UIAlertController prompt — exercises text entry into an + // alert field (which HID keystrokes can't reach; host-AX set-value can). + @State private var showGoToPage = false + @State private var pageInput = "" + @State private var goResult = "no page yet" var body: some View { NavigationView { Form { + Section("Go to Page") { + Button("Go to Page…") { showGoToPage = true } + .accessibilityIdentifier("btn_go_to_page") + Text(goResult) + .accessibilityIdentifier("lbl_go_result") + } Section("Identity") { TextField("First Name", text: $firstName) .accessibilityIdentifier("field_first_name") @@ -47,6 +58,12 @@ struct FormTab: View { } } .navigationTitle("TestKit") + .alert("Go to Page", isPresented: $showGoToPage) { + TextField("Enter a page number", text: $pageInput) + .accessibilityIdentifier("field_go_to_page") + Button("Go") { goResult = "Went to page \(pageInput)" } + Button("Cancel", role: .cancel) { } + } } } } diff --git a/simdrive/CHANGELOG.md b/simdrive/CHANGELOG.md index 7b06cdb1..88bc8ce1 100644 --- a/simdrive/CHANGELOG.md +++ b/simdrive/CHANGELOG.md @@ -13,6 +13,20 @@ filter catch what slips through. --> +## [1.0.0b10] — 2026-06-22 + +### Added — text entry into alert fields + accessibility journeys + +- **`set_text`** — set a text field's value directly via the host Accessibility + API, for fields that `type_text` (synthesized keystrokes) can't reach — most + importantly **`UIAlertController` prompts** (e.g. a "Go to Page" dialog). The + value propagates to the field's binding, so the app receives it. Targets the + field by `identifier`/`label`, or defaults to the alert's field. Simulator-only. +- **Journeys** can now use the accessibility tools: a `perform_accessibility_action` + step and an **`announcement_heard`** success criterion, so an end-to-end + VoiceOver flow (drive → invoke a custom action → assert the spoken announcement) + is expressible as a journey. + ## [1.0.0b9] — 2026-06-22 ### Added — accessibility automation (custom actions + VoiceOver announcements) diff --git a/simdrive/pyproject.toml b/simdrive/pyproject.toml index 0119ae5d..53e274fe 100644 --- a/simdrive/pyproject.toml +++ b/simdrive/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "simdrive" -version = "1.0.0b9" +version = "1.0.0b10" description = "SimDrive — reproduce and validate iOS bugs in 60 seconds with Claude. MCP-native iOS automation; 32 vision-first tools; record once, replay free in CI. 14-day trial, then $29/mo Pro." readme = "README.md" license = "Elastic-2.0" diff --git a/simdrive/src/simdrive/ax.py b/simdrive/src/simdrive/ax.py index 51500e60..9e7812f9 100644 --- a/simdrive/src/simdrive/ax.py +++ b/simdrive/src/simdrive/ax.py @@ -334,6 +334,60 @@ def perform_action( return {"ok": True, "action": name} +_EDITABLE_ROLES = {"AXTextField", "AXSecureTextField", "AXTextArea"} + + +def _find_text_field(root, depth=0, maxdepth=60): + if depth > maxdepth: + return None + if str(_attr(root, "AXRole") or "") in _EDITABLE_ROLES: + return root + for child in _children(root): + hit = _find_text_field(child, depth + 1, maxdepth) + if hit is not None: + return hit + return None + + +def set_text( + device_name: str, + text: str, + *, + identifier: str | None = None, + label: str | None = None, +) -> dict[str, Any]: + """Set a text field's value directly via host AX (`AXValue`). + + The fix for fields HID `type_text` can't reach — notably `UIAlertController` + prompts (e.g. a "Go to Page" dialog), whose field never receives synthesized + keystrokes. Setting `AXValue` propagates to the field's binding (verified: + the app reads the value), so the app receives the input. + + Resolution: scope to the device's window, then the field by *identifier* / + *label*, else the first editable field (text field / secure field / text + area) in the window — which is the alert's field when a prompt is up. + + Returns ``{"ok": True, "value": text}`` or ``{"ok": False, "error": ...}``. + """ + from ApplicationServices import AXUIElementSetAttributeValue # type: ignore[import] + + window = select_window(device_name) + if identifier: + field = _find_by(window, "AXIdentifier", identifier) + elif label: + field = _find_by(window, "AXDescription", label) or _find_by(window, "AXTitle", label) + else: + field = _find_text_field(window) + + if field is None: + return {"ok": False, "error": "no editable text field found in the target window"} + + err = AXUIElementSetAttributeValue(field, "AXValue", text) + if err != 0: + return {"ok": False, "error": f"AXUIElementSetAttributeValue err={err}"} + return {"ok": True, "value": text} + + # --------------------------------------------------------------------------- # Announcement observer (module singleton — one Simulator process) # --------------------------------------------------------------------------- diff --git a/simdrive/src/simdrive/journey/prompt.py b/simdrive/src/simdrive/journey/prompt.py index 6a4ca225..c0b46ff6 100644 --- a/simdrive/src/simdrive/journey/prompt.py +++ b/simdrive/src/simdrive/journey/prompt.py @@ -46,7 +46,7 @@ - The list of success criteria still unmet You must respond with a JSON object with these fields: - tool: one of "tap" | "swipe" | "type_text" | "press_key" | "clear_field" | "perform_accessibility_action" | "done" | "fail" + tool: one of "tap" | "swipe" | "type_text" | "press_key" | "clear_field" | "perform_accessibility_action" | "set_text" | "done" | "fail" args: a dict of arguments for the chosen tool (empty dict for "done"/"fail") rationale: one sentence explaining WHY you chose this action confidence: float 0.0-1.0 for how confident you are this action advances the goal @@ -59,6 +59,11 @@ are invisible to the screenshot; use them when a goal/criterion refers to a VoiceOver rotor action or a spoken announcement. +Use "set_text" (simulator only) to enter text into a field that "type_text" +can't reach — notably a UIAlertController prompt (e.g. "Go to Page"). args +{{"text": "..."}} (add {{"identifier": "..."}} or {{"label": "..."}} to target a +specific field; otherwise the alert's field is used). + Prefer stable_id over pixel coordinates when stable_ids are visible in the marks. Be precise with type_text — only type text explicitly required by the goal. Never guess or hallucinate element positions. diff --git a/simdrive/src/simdrive/journey/runner.py b/simdrive/src/simdrive/journey/runner.py index c38e6fe6..ca7b0bea 100644 --- a/simdrive/src/simdrive/journey/runner.py +++ b/simdrive/src/simdrive/journey/runner.py @@ -93,6 +93,11 @@ def tool_get_announcements(arguments: dict) -> dict: # pragma: no cover from simdrive.server import tool_get_announcements as _fn return _fn(arguments) + +def tool_set_text(arguments: dict) -> dict: # pragma: no cover + from simdrive.server import tool_set_text as _fn + return _fn(arguments) + # Approximate cost per LLM call (vision, claude-3-7 range); used for # in-flight cost tracking when the real client doesn't report token counts. _APPROX_COST_PER_CALL_USD = 0.004 @@ -110,7 +115,7 @@ class StepDecision: tool: Literal[ "tap", "swipe", "type_text", "press_key", "clear_field", - "perform_accessibility_action", "done", "fail", + "perform_accessibility_action", "set_text", "done", "fail", ] args: dict rationale: str @@ -219,6 +224,7 @@ def _dispatch_action(decision: StepDecision, session_id: str) -> None: "press_key": _self.tool_press_key, "clear_field": _self.tool_clear_field, "perform_accessibility_action": _self.tool_perform_accessibility_action, + "set_text": _self.tool_set_text, } tool_fn = tool_map.get(decision.tool) diff --git a/simdrive/src/simdrive/server.py b/simdrive/src/simdrive/server.py index 3d9e5070..2447dc10 100644 --- a/simdrive/src/simdrive/server.py +++ b/simdrive/src/simdrive/server.py @@ -1982,6 +1982,35 @@ def tool_perform_accessibility_action(arguments: dict) -> dict: return result +def tool_set_text(arguments: dict) -> dict: + """Set a text field's value directly via host AX (sim-only). + + For fields HID `type_text` can't reach — e.g. `UIAlertController` prompts. + """ + _entitlement_gate() + s = session.get(arguments["session_id"]) + if s.target != "simulator": + raise errors.invalid_argument( + "target", s.target, "set_text is simulator-only (host AX); use type_text on device" + ) + text = arguments.get("text") + if text is None: + raise errors.invalid_argument("text", text, "text is required") + try: + result = ax.set_text( + s.device.name, + str(text), + identifier=arguments.get("identifier"), + label=arguments.get("label"), + ) + except ax.AXError as exc: + raise errors.invalid_argument("session_id", arguments["session_id"], str(exc)) from exc + s.last_action_at = _now() + session.append_action(s, {"action": "set_text", "args": dict(arguments), + "result": result, "at": _now()}) + return result + + def tool_get_announcements(arguments: dict) -> dict: """Return VoiceOver announcements captured by the host AX observer (sim-only).""" _entitlement_gate() @@ -2368,6 +2397,28 @@ def tool_load_journey(arguments: dict) -> dict: }, "handler": tool_perform_accessibility_action, }, + { + "name": "set_text", + "description": ( + "(sim only) Set a text field's value directly via the host Accessibility " + "API — for fields that HID `type_text` can't reach, notably " + "UIAlertController prompts (e.g. a 'Go to Page' dialog). Targets the field " + "by `identifier`/`label`, else the first editable field in the window " + "(which is the alert's field when a prompt is up). The value propagates to " + "the field's binding, so the app receives it. Returns {ok, value}." + ), + "inputSchema": { + "type": "object", + "required": ["session_id", "text"], + "properties": { + "session_id": {"type": "string"}, + "text": {"type": "string", "description": "Text to set into the field."}, + "identifier": {"type": "string", "description": "Optional accessibilityIdentifier of the target field."}, + "label": {"type": "string", "description": "Optional accessibility label/description of the target field."}, + }, + }, + "handler": tool_set_text, + }, { "name": "get_announcements", "description": ( diff --git a/simdrive/tests/test_ax_module.py b/simdrive/tests/test_ax_module.py index dc85539d..cf6c0b40 100644 --- a/simdrive/tests/test_ax_module.py +++ b/simdrive/tests/test_ax_module.py @@ -39,6 +39,21 @@ def __init__(self, actions=None, children=None): self.children = children or [] +def test_find_text_field_recurses(monkeypatch): + class FE: + def __init__(self, role="", children=None): + self.role = role + self.children = children or [] + + monkeypatch.setattr(ax, "_attr", lambda e, a: e.role if a == "AXRole" else None) + monkeypatch.setattr(ax, "_children", lambda e: e.children) + + field = FE(role="AXTextField") + root = FE(children=[FE(role="AXButton"), FE(children=[field])]) + assert ax._find_text_field(root) is field + assert ax._find_text_field(FE(children=[FE(role="AXButton")])) is None + + def test_find_action_carrier_recurses_to_deep_child(monkeypatch): monkeypatch.setattr(ax, "_action_names", lambda e: e.actions) monkeypatch.setattr(ax, "_children", lambda e: e.children) @@ -116,7 +131,7 @@ def test_ax_tools_registered(): from simdrive import server names = {t["name"] for t in server._TOOLS} - assert {"perform_accessibility_action", "get_announcements"} <= names + assert {"perform_accessibility_action", "get_announcements", "set_text"} <= names # handlers are callable and listed without the handler key in list_tools() listed = {t["name"] for t in server.list_tools()} - assert {"perform_accessibility_action", "get_announcements"} <= listed + assert {"perform_accessibility_action", "get_announcements", "set_text"} <= listed diff --git a/simdrive/tests/test_journey_ax.py b/simdrive/tests/test_journey_ax.py index c42ad40c..e462a267 100644 --- a/simdrive/tests/test_journey_ax.py +++ b/simdrive/tests/test_journey_ax.py @@ -73,6 +73,20 @@ def test_dispatch_routes_perform_accessibility_action(): def test_step_decision_accepts_new_tool_literal(): - # Constructing with the new tool name must not raise. + # Constructing with the new tool names must not raise. d = StepDecision(tool="perform_accessibility_action", args={}, rationale="r", confidence=0.5) assert d.tool == "perform_accessibility_action" + d2 = StepDecision(tool="set_text", args={"text": "42"}, rationale="r", confidence=0.5) + assert d2.tool == "set_text" + + +def test_dispatch_routes_set_text(): + decision = StepDecision( + tool="set_text", args={"text": "42"}, rationale="enter page", confidence=0.9 + ) + with patch("simdrive.journey.runner.tool_set_text", return_value={"ok": True}) as fn: + _dispatch_action(decision, "sess-2") + fn.assert_called_once() + passed = fn.call_args[0][0] + assert passed["text"] == "42" + assert passed["session_id"] == "sess-2" diff --git a/simdrive/tests/test_paywall_gates.py b/simdrive/tests/test_paywall_gates.py index fdcc2623..9d4da40f 100644 --- a/simdrive/tests/test_paywall_gates.py +++ b/simdrive/tests/test_paywall_gates.py @@ -23,7 +23,7 @@ # The full canonical 32-tool registry — sourced from server._TOOLS at runtime. # A test below pins this count so adding/removing tools without updating the # gate is caught immediately. -EXPECTED_TOOL_COUNT = 35 # +2: perform_accessibility_action, get_announcements (host-AX a11y) +EXPECTED_TOOL_COUNT = 36 # +3: perform_accessibility_action, get_announcements, set_text (host-AX a11y) # --------------------------------------------------------------------------- @@ -131,6 +131,7 @@ def test_tool_count_pinned_at_32(self) -> None: "load_journey", "perform_accessibility_action", "get_announcements", + "set_text", ] diff --git a/simdrive/tests/test_unit.py b/simdrive/tests/test_unit.py index 12a8c863..a3ad3f5a 100644 --- a/simdrive/tests/test_unit.py +++ b/simdrive/tests/test_unit.py @@ -40,7 +40,7 @@ def test_tool_count_is_thirty_two(): + lint_recordings + migrate_recording (a9.1) = 32 """ tools = server.list_tools() - assert len(tools) == 35, f"expected 35 tools, got {len(tools)}: {[t['name'] for t in tools]}" + assert len(tools) == 36, f"expected 36 tools, got {len(tools)}: {[t['name'] for t in tools]}" def test_tool_names_match_spec(): @@ -65,6 +65,8 @@ def test_tool_names_match_spec(): "tap_and_wait_keyboard", # Host-AX accessibility (PP-4527): custom actions + VoiceOver announcements "perform_accessibility_action", "get_announcements", + # Host-AX text entry for fields HID can't reach (UIAlertController) + "set_text", } got = {t["name"] for t in server.list_tools()} assert got == expected, f"missing: {expected - got}, extra: {got - expected}"