fix(ios): orientation handling so landscape devices work

Author: NathanWalker

packages/client/src/features/accessibility/AccessibilityOverlay.test.ts

@@ -129,4 +129,45 @@ describe("AccessibilityOverlay", () => {
     expect(markup).toContain("accessibility-rect skeleton");
     expect(markup).not.toContain("<span>Continue</span>");
   });
+
+  it("converts display-space frames into natural space for a rotated landscape display", () => {
+    // Landscape display (1210x834). The child is the right half of the display
+    // in display space; after the shell's 270deg rotation it must land in the
+    // bottom half of the natural (portrait) presentation box.
+    const roots = [
+      {
+        frame: { height: 834, width: 1210, x: 0, y: 0 },
+        role: "application",
+        children: [
+          {
+            AXLabel: "RightHalf",
+            frame: { height: 834, width: 605, x: 605, y: 0 },
+            type: "Button",
+          },
+        ],
+      },
+    ];
+
+    const rotated = renderToStaticMarkup(
+      createElement(AccessibilityOverlay, {
+        hoveredId: null,
+        roots,
+        rotationQuarterTurns: 3,
+        selectedId: "",
+      }),
+    );
+    // display right-half -> natural bottom-half.
+    expect(rotated).toContain("height:50%;left:0%;top:50%;width:100%");
+
+    const unrotated = renderToStaticMarkup(
+      createElement(AccessibilityOverlay, {
+        hoveredId: null,
+        roots,
+        rotationQuarterTurns: 0,
+        selectedId: "",
+      }),
+    );
+    // With no rotation the frame keeps its display-space placement (right half).
+    expect(unrotated).toContain("height:100%;left:50%;top:0%;width:50%");
+  });
 });

packages/client/src/features/accessibility/AccessibilityOverlay.tsx

@@ -1,6 +1,7 @@
 import { createElement, type AriaRole, type CSSProperties } from "react";
 
 import type { AccessibilityNode } from "../../api/types";
+import { mapDisplayedPointToNaturalOrientation } from "../viewport/viewportMath";
 import {
   accessibilityKind,
   accessibilityIdentifier,
@@ -16,13 +17,15 @@ import {
 interface AccessibilityOverlayProps {
   hoveredId: string | null;
   roots: AccessibilityNode[];
+  rotationQuarterTurns?: number;
   selectedId: string;
   skeletonVisible?: boolean;
 }
 
 export function AccessibilityOverlay({
   hoveredId,
   roots,
+  rotationQuarterTurns = 0,
   selectedId,
   skeletonVisible = false,
 }: AccessibilityOverlayProps) {
@@ -61,6 +64,7 @@ export function AccessibilityOverlay({
             key={item.id}
             node={item.node}
             rootFrame={rootFrame}
+            rotationQuarterTurns={rotationQuarterTurns}
           />
         ))}
       </div>
@@ -71,15 +75,26 @@ export function AccessibilityOverlay({
                 key={`skeleton-${item.id}`}
                 node={item.node}
                 rootFrame={rootFrame}
+                rotationQuarterTurns={rotationQuarterTurns}
                 variant="skeleton"
               />
             ))
           : null}
         {hovered ? (
-          <NodeRect node={hovered} rootFrame={rootFrame} variant="hovered" />
+          <NodeRect
+            node={hovered}
+            rootFrame={rootFrame}
+            rotationQuarterTurns={rotationQuarterTurns}
+            variant="hovered"
+          />
         ) : null}
         {selected ? (
-          <NodeRect node={selected} rootFrame={rootFrame} variant="selected" />
+          <NodeRect
+            node={selected}
+            rootFrame={rootFrame}
+            rotationQuarterTurns={rotationQuarterTurns}
+            variant="selected"
+          />
         ) : null}
       </div>
     </div>
@@ -107,20 +122,18 @@ function framedNode(
 function NodeRect({
   node,
   rootFrame,
+  rotationQuarterTurns,
   variant,
 }: {
   node: AccessibilityNode;
   rootFrame: { height: number; width: number; x: number; y: number };
+  rotationQuarterTurns: number;
   variant: "hovered" | "selected" | "skeleton";
 }) {
   if (!validFrame(node.frame)) {
     return null;
   }
 
-  const left = ((node.frame.x - rootFrame.x) / rootFrame.width) * 100;
-  const top = ((node.frame.y - rootFrame.y) / rootFrame.height) * 100;
-  const width = (node.frame.width / rootFrame.width) * 100;
-  const height = (node.frame.height / rootFrame.height) * 100;
   const label =
     variant === "skeleton"
       ? ""
@@ -129,12 +142,7 @@ function NodeRect({
   return (
     <div
       className={`accessibility-rect ${variant}`}
-      style={{
-        height: `${height}%`,
-        left: `${left}%`,
-        top: `${top}%`,
-        width: `${width}%`,
-      }}
+      style={frameStyle(node.frame, rootFrame, rotationQuarterTurns)}
     >
       {label ? <span>{label}</span> : null}
     </div>
@@ -146,11 +154,13 @@ function AccessibilityDomNode({
   id,
   node,
   rootFrame,
+  rotationQuarterTurns,
 }: {
   depth: number;
   id: string;
   node: AccessibilityNode;
   rootFrame: { height: number; width: number; x: number; y: number };
+  rotationQuarterTurns: number;
 }) {
   if (!validFrame(node.frame)) {
     return null;
@@ -200,19 +210,40 @@ function AccessibilityDomNode({
     "data-simdeck-inspector-id": node.inspectorId || undefined,
     "data-simdeck-uikit-id": node.uikitId || undefined,
     role,
-    style: frameStyle(node.frame, rootFrame),
+    style: frameStyle(node.frame, rootFrame, rotationQuarterTurns),
   });
 }
 
+// Native-ax frames arrive in DISPLAY (interface) space. The overlay lives
+// inside the shell container that CSS-rotates by `rotationQuarterTurns`, so the
+// frame is converted back into the device's natural (pre-rotation) space here;
+// the shared shell rotation then carries the overlay and the video to the
+// displayed orientation together. `rotationQuarterTurns === 0` is the identity,
+// so portrait devices keep their existing behavior.
 function frameStyle(
   frame: { height: number; width: number; x: number; y: number },
   rootFrame: { height: number; width: number; x: number; y: number },
+  rotationQuarterTurns: number,
 ): CSSProperties {
+  const start = mapDisplayedPointToNaturalOrientation(
+    {
+      x: (frame.x - rootFrame.x) / rootFrame.width,
+      y: (frame.y - rootFrame.y) / rootFrame.height,
+    },
+    rotationQuarterTurns,
+  );
+  const end = mapDisplayedPointToNaturalOrientation(
+    {
+      x: (frame.x + frame.width - rootFrame.x) / rootFrame.width,
+      y: (frame.y + frame.height - rootFrame.y) / rootFrame.height,
+    },
+    rotationQuarterTurns,
+  );
   return {
-    height: `${(frame.height / rootFrame.height) * 100}%`,
-    left: `${((frame.x - rootFrame.x) / rootFrame.width) * 100}%`,
-    top: `${((frame.y - rootFrame.y) / rootFrame.height) * 100}%`,
-    width: `${(frame.width / rootFrame.width) * 100}%`,
+    height: `${Math.abs(end.y - start.y) * 100}%`,
+    left: `${Math.min(start.x, end.x) * 100}%`,
+    top: `${Math.min(start.y, end.y) * 100}%`,
+    width: `${Math.abs(end.x - start.x) * 100}%`,
   };
 }
 

packages/client/src/features/viewport/DeviceChrome.tsx

@@ -7,7 +7,7 @@ import type {
 } from "../../api/types";
 import { AccessibilityOverlay } from "../accessibility/AccessibilityOverlay";
 import { findAccessibilityItemAtPoint } from "../accessibility/accessibilityTree";
-import { normalizedPointerCoordinatesForOrientation } from "../input/gestureMath";
+import { normalizedPointerCoordinates } from "../input/gestureMath";
 import type { TouchIndicator } from "./types";
 
 interface DeviceChromeProps {
@@ -698,6 +698,7 @@ function ScreenLayer({
       <AccessibilityOverlay
         hoveredId={accessibilityHoveredId}
         roots={accessibilityRoots}
+        rotationQuarterTurns={rotationQuarterTurns}
         selectedId={accessibilitySelectedId}
         skeletonVisible={accessibilitySkeletonVisible}
       />
@@ -719,22 +720,12 @@ function ScreenLayer({
           onPointerMove={(event) => {
             event.preventDefault();
             event.stopPropagation();
-            onPickerHover(
-              hitTestAccessibilityId(
-                event,
-                accessibilityRoots,
-                rotationQuarterTurns,
-              ),
-            );
+            onPickerHover(hitTestAccessibilityId(event, accessibilityRoots));
           }}
           onPointerUp={(event) => {
             event.preventDefault();
             event.stopPropagation();
-            const id = hitTestAccessibilityId(
-              event,
-              accessibilityRoots,
-              rotationQuarterTurns,
-            );
+            const id = hitTestAccessibilityId(event, accessibilityRoots);
             if (id) {
               onPickerSelect(id);
             }
@@ -788,15 +779,16 @@ function TouchInteractionOverlay({
   );
 }
 
+// Native-ax frames are in DISPLAY (interface) space, so the picker compares the
+// pointer directly in displayed coordinates — no display→natural remap. (The
+// overlay does the inverse remap for rendering because it lives inside the
+// CSS-rotated shell; the pointer here is already measured against the displayed,
+// post-rotation bounding box.)
 function hitTestAccessibilityId(
   event: React.PointerEvent<HTMLElement>,
   roots: AccessibilityNode[],
-  rotationQuarterTurns: number,
 ): string | null {
-  const point = normalizedPointerCoordinatesForOrientation(
-    event,
-    rotationQuarterTurns,
-  );
+  const point = normalizedPointerCoordinates(event);
   if (!point) {
     return null;
   }