From 8163a4e5646b1243e39c33d20fefa504e0562801 Mon Sep 17 00:00:00 2001 From: Maurice Carrier Date: Tue, 23 Jun 2026 14:45:14 -0400 Subject: [PATCH] fix(ax): restore iOS-26 content-group position-probe for host-AX find paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The specterqa→simdrive rename dropped the iOS content-group position-probe from the host-AX backend. On iOS 26 the Simulator collapses the app UI into a single opaque AXGroup (subrole "iOSContentGroup") with zero AXChildren, so the plain window tree-walk in `set_text` and `perform_action` sees nothing inside the app — alert text fields and custom-action carriers report "not found". Port the heuristics back into the flat `simdrive/ax.py` (legacy was a class): - `_frame` — read AXFrame as a CGRect via AXValueGetValue (lazy pyobjc import). - `_find_ios_content_group` — aspect-ratio heuristic (largest portrait child). - `_position_probe_content_group` — AXUIElementCopyElementAtPosition at the device window's screen-centre, then walk up to the nearest group container. This resolves real content even when AXChildren is empty (iOS 26 path). - `_resolve_content_group` — heuristic first, but fall through to the probe when the matched group is itself childless (the iOS-26 opaque shape), so the fallback is actually walkable. Wire into `set_text` (`_find_text_field`/`_find_by`) and `perform_action` (`_find_action_carrier`, plus the WKWebView-hint accuracy check): only when the plain window walk finds nothing do we resolve + search the content-group subtree. Non-iOS-26 paths are unchanged. Live-proved on a booted iOS 26.0 sim (iPhone 16 Pro) running Palace/A1QA: - set_text: with the "Go to Page" UIAlertController up, the alert's AXTextField is now resolved + settable (b11 returned "no editable text field found"). - perform_action: the reader's native custom actions ("Move next/previous") are surfaced and performable; web-content actions like "Where am I?" remain unreachable (Readium/WKWebView a11y is not bridged to host-AX — documented). Scar-tissue / critical-path code (ax.py) — heuristic ported faithfully, not "improved". Operator-authorized as the dedicated ticket per CLAUDE.md. **Scope:** content-group probe port + wiring + unit tests + version/changelog. **Not done:** no new tool surface; no XCTest-backend web-content a11y path. **Deferred:** enabling on-device accessibility from simdrive (the host-AX bridge requires it active in the sim) — out of scope for this regression fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- simdrive/CHANGELOG.md | 23 +++ simdrive/pyproject.toml | 2 +- simdrive/src/simdrive/ax.py | 199 ++++++++++++++++++++++++- simdrive/tests/test_ax_module.py | 242 +++++++++++++++++++++++++++++++ 4 files changed, 463 insertions(+), 3 deletions(-) diff --git a/simdrive/CHANGELOG.md b/simdrive/CHANGELOG.md index 18c4f70d..8454c8ec 100644 --- a/simdrive/CHANGELOG.md +++ b/simdrive/CHANGELOG.md @@ -13,6 +13,29 @@ filter catch what slips through. --> +## [1.0.0b12] — 2026-06-23 + +### Fixed — host-AX `set_text` / `perform_accessibility_action` now reach app content on iOS 26 + +- **Restored the iOS content-group position-probe.** On iOS 26 the Simulator + collapses the whole app UI into a single opaque accessibility group that + reports no children, so the host-AX tree-walk used by `set_text` and + `perform_accessibility_action` saw nothing inside the app — text fields (e.g. + a "Go to Page" alert) and custom-action carriers were reported as "not + found". simdrive now resolves that content group via an aspect-ratio + heuristic and, when the group is opaque, an `AXUIElementCopyElementAtPosition` + position-probe at the device window's centre — the same iOS-26-compatible + approach the engine shipped before. When the plain window walk finds nothing, + both tools now search the resolved content-group subtree before giving up. +- Non-iOS-26 paths are unchanged: when the plain window walk already finds the + field or action, the content-group fallback is never consulted. +- **Boundary note:** Readium/WKWebView reader *web-content* actions (e.g. + "Where am I?") remain unreachable from host-AX — web-content accessibility is + not bridged to the macOS AX tree on the simulator. The probe surfaces the + reader's native chrome actions ("Move next/previous"); for web-content a11y, + use the on-device XCTest backend. The host-AX bridge also requires on-device + accessibility to be active in the simulator. + ## [1.0.0b11] — 2026-06-23 ### Fixed — MCP auto-restart no longer breaks the stdio session diff --git a/simdrive/pyproject.toml b/simdrive/pyproject.toml index 93c9021a..93503c13 100644 --- a/simdrive/pyproject.toml +++ b/simdrive/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "simdrive" -version = "1.0.0b11" +version = "1.0.0b12" description = "SimDrive — reproduce and validate iOS bugs in 60 seconds with Claude. MCP-native iOS automation; 32 vision-first tools; record once, replay free in CI. 14-day trial, then $29/mo Pro." readme = "README.md" license = "Elastic-2.0" diff --git a/simdrive/src/simdrive/ax.py b/simdrive/src/simdrive/ax.py index f51b2dbc..bdeb2821 100644 --- a/simdrive/src/simdrive/ax.py +++ b/simdrive/src/simdrive/ax.py @@ -96,6 +96,49 @@ def _children(elem: Any) -> list[Any]: return [kids] +def _frame(elem: Any) -> dict | None: + """Return *elem*'s AXFrame as ``{x, y, width, height}`` in macOS screen pts. + + Reads the ``AXFrame`` attribute (a ``CGRect``-bearing ``AXValue``) via + ``AXValueGetValue`` with ``kAXValueCGRectType``. Returns ``None`` when the + frame can't be read. pyobjc/ApplicationServices imports stay lazy. + """ + try: + from ApplicationServices import ( # type: ignore[import] + AXUIElementCopyAttributeValue, + AXValueGetValue, + kAXValueCGRectType, + ) + + err, val = AXUIElementCopyAttributeValue(elem, "AXFrame", None) + if err != 0 or val is None: + return None + ok, rect = AXValueGetValue(val, kAXValueCGRectType, None) + if not ok or rect is None: + return None + try: + return { + "x": float(rect.origin.x), + "y": float(rect.origin.y), + "width": float(rect.size.width), + "height": float(rect.size.height), + } + except AttributeError: + # Some pyobjc versions return a (x, y, w, h) tuple. + if hasattr(rect, "__iter__"): + vals = list(rect) + if len(vals) == 4: + return { + "x": float(vals[0]), + "y": float(vals[1]), + "width": float(vals[2]), + "height": float(vals[3]), + } + except Exception as exc: # noqa: BLE001 + logger.debug("_frame failed: %s", exc) + return None + + def _action_names(elem: Any) -> list[str]: from ApplicationServices import AXUIElementCopyActionNames # type: ignore[import] @@ -240,6 +283,126 @@ def _match() -> Any: ) +# --------------------------------------------------------------------------- +# iOS content-group resolution (iOS 26 host-AX compatibility) +# +# Ported faithfully from the legacy class-based AXBackend +# (src/specterqa/ios/backends/ax_backend.py: _find_ios_content_group, +# _position_probe_content_group, _init_content_group). On iOS 26 the +# Simulator collapses the app UI into a single AXGroup with subrole +# "iOSContentGroup" that exposes ZERO children via AXChildren / +# AXChildrenInNavigationOrder / AXVisibleChildren — so a plain window +# tree-walk sees nothing inside the app. The fix is a two-stage resolve: +# +# 1. aspect-ratio heuristic: the largest descendant AXGroup whose frame +# matches an iOS portrait screen (w/h ~0.35–0.75). +# 2. position-probe fallback (iOS 26 compatible): hit-test the window's +# screen-centre with AXUIElementCopyElementAtPosition, which returns +# the REAL element under the point even when AXChildren is empty, then +# walk up to the nearest sensible container. +# +# Scar tissue — AX-tree shape is brittle across iOS sims; do not "improve" +# the heuristic (CLAUDE.md). Adapted class→flat: no instance cache; the +# resolver takes the already-selected window and returns an element whose +# subtree a DFS can search. +# --------------------------------------------------------------------------- + + +def _find_ios_content_group(window: Any) -> Any | None: + """Aspect-ratio heuristic: the iOS screen group inside *window*. + + Walks the window's direct children for the largest one whose frame matches + an iOS portrait aspect ratio (0.35 < w/h < 0.75). Mirrors the legacy + ``_find_ios_content_group`` but scoped to a single already-selected window + (the flat module resolves the window up front via :func:`select_window`). + + Returns the content-group AXUIElement, or ``None`` when no child matches. + """ + best_elem: Any = None + best_area = 0.0 + for child in _children(window): + cf = _frame(child) + if cf is None: + continue + w = cf["width"] + h = cf["height"] + area = w * h + if area > best_area and h > 0 and (0.35 < w / h < 0.75): + best_area = area + best_elem = child + return best_elem + + +def _position_probe_content_group(window: Any) -> Any | None: + """Fallback: probe the AX element at *window*'s screen centre. + + Uses ``AXUIElementCopyElementAtPosition`` (the same position API the legacy + backend used for the tab bar) to hit-test the centre of the window's frame. + On iOS 26 the ``iOSContentGroup`` AXGroup reports zero AXChildren, but a + position hit-test still resolves the real element drawn under the point — + so the returned element's own subtree IS walkable. We then walk up the + parent chain to the nearest AXGroup/AXWindow container (depth-capped) so a + DFS from it covers the whole presented surface. + + Returns the resolved container element, or ``None`` when unavailable. + """ + try: + from ApplicationServices import ( # type: ignore[import] + AXUIElementCopyElementAtPosition, + ) + except ImportError: + logger.debug("position-probe unavailable: ApplicationServices not importable") + return None + + frame = _frame(window) + if frame is None or frame["width"] <= 0 or frame["height"] <= 0: + logger.debug("position-probe: could not read target window frame") + return None + + centre_x = frame["x"] + frame["width"] / 2.0 + centre_y = frame["y"] + frame["height"] / 2.0 + + app = _app_element() + err, hit = AXUIElementCopyElementAtPosition(app, centre_x, centre_y, None) + if err != 0 or hit is None: + logger.debug("position-probe: copyElementAtPosition err=%s", err) + return None + + # Walk up to the nearest window-level / group container (cap depth to avoid + # loops on malformed trees), matching the legacy probe's parent-walk. + candidate = hit + for _ in range(10): + role = str(_attr(candidate, "AXRole") or "") + if role in ("AXWindow", "AXGroup"): + break + parent = _attr(candidate, "AXParent") + if parent is None: + break + candidate = parent + return candidate + + +def _resolve_content_group(window: Any) -> Any | None: + """Resolve the iOS content group for *window* (heuristic → position-probe). + + Returns an element whose subtree a DFS can search to reach the app's real + accessibility content on iOS 26 (where the window walk alone sees an opaque, + childless ``iOSContentGroup``). ``None`` when neither stage resolves a group. + """ + try: + elem = _find_ios_content_group(window) + # Only trust the heuristic when the matched group is actually walkable. + # On iOS 26 the aspect-ratio match is frequently the opaque, CHILDLESS + # ``iOSContentGroup`` itself — returning it would dead-end a DFS, so we + # fall through to the position-probe (the iOS-26 expansion path) in that + # case rather than "improving" the brittle heuristic. + if elem is not None and _children(elem): + return elem + except Exception as exc: # noqa: BLE001 + logger.debug("content-group heuristic failed: %s — trying position-probe", exc) + return _position_probe_content_group(window) + + # --------------------------------------------------------------------------- # Element + action-carrier resolution # --------------------------------------------------------------------------- @@ -348,12 +511,29 @@ def perform_action( # Explicit target didn't carry it — widen to the whole window. carrier = _find_action_carrier(window, name) + if carrier is None: + # iOS 26: the plain window walk can bottom out at an opaque, childless + # ``iOSContentGroup`` AXGroup. Resolve the content group via the + # heuristic/position-probe and DFS its subtree — this is what surfaces + # the real custom-action carriers (e.g. the reader's "Where am I?"). + group = _resolve_content_group(window) + if group is not None and group is not window: + carrier = _find_action_carrier(group, name) + if carrier is None: error = f"custom action {name!r} not found in the target window" # b11 FIX 2: if the window has NO custom actions at all (not merely a # missing named one), hint at the WKWebView/Readium host-AX boundary. - # Keep it accurate — only when zero actions exist window-wide. - if not _window_has_any_custom_action(window): + # Keep it accurate — only when zero actions exist window-wide. iOS 26: + # also probe the resolved content-group subtree (the window walk alone + # can't see past an opaque ``iOSContentGroup``) before concluding zero. + group = _resolve_content_group(window) + has_actions = _window_has_any_custom_action(window) or ( + group is not None + and group is not window + and _window_has_any_custom_action(group) + ) + if not has_actions: error += _WKWEBVIEW_HINT return { "ok": False, @@ -425,6 +605,21 @@ def set_text( else: field = _find_text_field(window) + if field is None: + # iOS 26: the plain window walk can bottom out at an opaque, childless + # ``iOSContentGroup`` AXGroup. Resolve the content group via the + # heuristic/position-probe and search its subtree for the field. + group = _resolve_content_group(window) + if group is not None and group is not window: + if identifier: + field = _find_by(group, "AXIdentifier", identifier) + elif label: + field = _find_by(group, "AXDescription", label) or _find_by( + group, "AXTitle", label + ) + else: + field = _find_text_field(group) + if field is None: return {"ok": False, "error": "no editable text field found in the target window"} diff --git a/simdrive/tests/test_ax_module.py b/simdrive/tests/test_ax_module.py index 1d185429..27c78f55 100644 --- a/simdrive/tests/test_ax_module.py +++ b/simdrive/tests/test_ax_module.py @@ -141,6 +141,248 @@ def test_set_text_docstring_documents_swiftui_boundary(): assert "type_text" in doc +# ── iOS-26 content-group probe (regression: dropped in specterqa→simdrive) ──── +# +# On iOS 26 the Simulator collapses the app UI into one opaque AXGroup +# (subrole "iOSContentGroup") with ZERO AXChildren, so the plain window walk +# in set_text/perform_action sees nothing inside the app. The ported +# heuristic + AXUIElementCopyElementAtPosition position-probe expand it. +# These tests catch the regression: WITHOUT the resolver the field/action is +# not found; WITH it, it is. + + +class _Node: + """Minimal fake AX element for the content-group tests.""" + + def __init__(self, role="", subrole="", frame=None, children=None, + actions=None, ident="", desc="", title=""): + self.role = role + self.subrole = subrole + self.frame = frame + self.children = children or [] + self.actions = actions or [] + self.ident = ident + self.desc = desc + self.title = title + + +def _wire_fakes(monkeypatch, *, hit=None): + """Route ax's low-level accessors at the _Node fakes.""" + def _attr(e, a): + return { + "AXRole": e.role, + "AXSubrole": e.subrole, + "AXIdentifier": e.ident, + "AXDescription": e.desc, + "AXTitle": e.title, + }.get(a) + + monkeypatch.setattr(ax, "_attr", _attr) + monkeypatch.setattr(ax, "_children", lambda e: e.children) + monkeypatch.setattr(ax, "_frame", lambda e: e.frame) + monkeypatch.setattr(ax, "_action_names", lambda e: e.actions) + + +# ── aspect-ratio heuristic ──────────────────────────────────────────────────── + + +def test_find_ios_content_group_picks_portrait_largest(monkeypatch): + # A wide chrome bar (landscape) + a portrait content group; the portrait, + # larger-area child wins. + chrome = _Node(role="AXToolbar", frame={"x": 0, "y": 0, "width": 400, "height": 40}) + content = _Node(role="AXGroup", subrole="iOSContentGroup", + frame={"x": 0, "y": 0, "width": 390, "height": 844}, + children=[_Node(role="AXTextField")]) + window = _Node(role="AXWindow", children=[chrome, content]) + _wire_fakes(monkeypatch) + assert ax._find_ios_content_group(window) is content + + +def test_find_ios_content_group_none_when_no_portrait_child(monkeypatch): + window = _Node(role="AXWindow", children=[ + _Node(role="AXButton", frame={"x": 0, "y": 0, "width": 400, "height": 40}), + ]) + _wire_fakes(monkeypatch) + assert ax._find_ios_content_group(window) is None + + +# ── position-probe fallback (AXUIElementCopyElementAtPosition) ──────────────── + + +def test_position_probe_hits_window_centre(monkeypatch): + # The probe hit-tests the window centre and walks up to a group container — + # this is the path that resolves real content when AXChildren is empty. + real = _Node(role="AXStaticText", desc="hello") + container = _Node(role="AXGroup", children=[real, _Node(role="AXTextField")]) + real.parent_container = container + + window = _Node(role="AXWindow", frame={"x": 100, "y": 100, "width": 400, "height": 800}) + _wire_fakes(monkeypatch) + monkeypatch.setattr(ax, "_app_element", lambda: object()) + + captured = {} + + def _copy_at(app, x, y, _none): + captured["xy"] = (x, y) + return 0, real + + # _attr must also vend AXParent for the up-walk. + base_attr = ax._attr + + def _attr_with_parent(e, a): + if a == "AXParent": + return getattr(e, "parent_container", None) + return base_attr(e, a) + + monkeypatch.setattr(ax, "_attr", _attr_with_parent) + monkeypatch.setitem( + __import__("sys").modules, + "ApplicationServices", + types.SimpleNamespace(AXUIElementCopyElementAtPosition=_copy_at), + ) + grp = ax._position_probe_content_group(window) + assert grp is container # walked up from the hit element to the group + assert captured["xy"] == (300.0, 500.0) # exact window centre + + +def test_position_probe_none_without_window_frame(monkeypatch): + window = _Node(role="AXWindow", frame=None) + _wire_fakes(monkeypatch) + monkeypatch.setattr(ax, "_app_element", lambda: object()) + monkeypatch.setitem( + __import__("sys").modules, + "ApplicationServices", + types.SimpleNamespace(AXUIElementCopyElementAtPosition=lambda *a: (0, None)), + ) + assert ax._position_probe_content_group(window) is None + + +# ── resolver: childless heuristic group MUST fall through to the probe ───────── + + +def test_resolve_content_group_falls_through_when_heuristic_childless(monkeypatch): + # The heuristic matches the opaque, CHILDLESS iOSContentGroup (the iOS-26 + # failure shape). The resolver must NOT return it — it must fall through to + # the position-probe, which surfaces the real (walkable) container. + childless = _Node(role="AXGroup", subrole="iOSContentGroup", + frame={"x": 0, "y": 0, "width": 390, "height": 844}, + children=[]) + window = _Node(role="AXWindow", frame={"x": 0, "y": 0, "width": 390, "height": 844}, + children=[childless]) + probed = _Node(role="AXGroup", children=[_Node(role="AXTextField")]) + _wire_fakes(monkeypatch) + monkeypatch.setattr(ax, "_position_probe_content_group", lambda w: probed) + assert ax._resolve_content_group(window) is probed + + +def test_resolve_content_group_trusts_walkable_heuristic(monkeypatch): + walkable = _Node(role="AXGroup", subrole="iOSContentGroup", + frame={"x": 0, "y": 0, "width": 390, "height": 844}, + children=[_Node(role="AXTextField")]) + window = _Node(role="AXWindow", children=[walkable]) + _wire_fakes(monkeypatch) + # Probe must NOT be consulted when the heuristic group is already walkable. + monkeypatch.setattr(ax, "_position_probe_content_group", + lambda w: pytest.fail("probe should not run")) + assert ax._resolve_content_group(window) is walkable + + +# ── wired-in regression: set_text finds the field only via the content group ── + + +def test_set_text_finds_field_via_content_group_probe(monkeypatch): + """iOS 26: field is invisible to the window walk, reached via the probe. + + BEFORE (no probe): _find_text_field(window) is None -> 'not found'. + AFTER (probe): the resolved content group exposes the AXTextField -> set. + """ + field = _Node(role="AXTextField") + window = _Node(role="AXWindow", children=[ + _Node(role="AXGroup", subrole="iOSContentGroup", children=[]), # opaque + ]) + content_group = _Node(role="AXGroup", children=[field]) + + _wire_fakes(monkeypatch) + monkeypatch.setattr(ax, "select_window", lambda dev: window) + monkeypatch.setattr(ax, "_resolve_content_group", lambda w: content_group) + + set_calls = {} + + def _set(el, attr, val): + set_calls["args"] = (el, attr, val) + return 0 + + monkeypatch.setitem( + __import__("sys").modules, + "ApplicationServices", + types.SimpleNamespace(AXUIElementSetAttributeValue=_set), + ) + + # Sanity: the plain window walk alone does NOT find the field (regression). + assert ax._find_text_field(window) is None + + res = ax.set_text("iPhone 16 Pro", "5") + assert res == {"ok": True, "value": "5"} + assert set_calls["args"] == (field, "AXValue", "5") + + +def test_set_text_still_not_found_without_content_group(monkeypatch): + """Regression guard: with NO resolvable content group, still 'not found'.""" + window = _Node(role="AXWindow", children=[ + _Node(role="AXGroup", subrole="iOSContentGroup", children=[]), + ]) + _wire_fakes(monkeypatch) + monkeypatch.setattr(ax, "select_window", lambda dev: window) + monkeypatch.setattr(ax, "_resolve_content_group", lambda w: None) + monkeypatch.setitem( + __import__("sys").modules, + "ApplicationServices", + types.SimpleNamespace(AXUIElementSetAttributeValue=lambda *a: 0), + ) + res = ax.set_text("iPhone 16 Pro", "5") + assert res["ok"] is False + assert "no editable text field" in res["error"] + + +# ── wired-in regression: perform_action reaches a carrier via the probe ─────── + + +def test_perform_action_finds_carrier_via_content_group_probe(monkeypatch): + """iOS 26: the carrier lives behind the opaque content group. + + BEFORE: the window walk finds no carrier -> 'not found'. + AFTER: the resolved content group exposes the carrier -> performed. + """ + carrier = _Node(actions=["Name:Where am I?\nTarget:0x0\nSelector:(null)"]) + window = _Node(role="AXWindow", children=[ + _Node(role="AXGroup", subrole="iOSContentGroup", children=[]), + ]) + content_group = _Node(role="AXGroup", children=[carrier]) + + _wire_fakes(monkeypatch) + monkeypatch.setattr(ax, "select_window", lambda dev: window) + monkeypatch.setattr(ax, "_resolve_content_group", lambda w: content_group) + + performed = {} + + def _perform(el, name): + performed["args"] = (el, name) + return 0 + + monkeypatch.setitem( + __import__("sys").modules, + "ApplicationServices", + types.SimpleNamespace(AXUIElementPerformAction=_perform), + ) + + # Sanity: the plain window walk alone finds no carrier (regression). + assert ax._find_action_carrier(window, "Where am I?") is None + + res = ax.perform_action("iPhone 16 Pro", "Where am I?") + assert res == {"ok": True, "action": "Where am I?"} + assert performed["args"][0] is carrier + + # ── announcement buffer: soft pid scoping (never drops to a false empty) ──────