fix(wait): surface runner restart timeout evidence

Author: kkkhs

packages/contracts/src/wait.ts

@@ -13,6 +13,7 @@
 export const WAIT_REASONS = {
   captureStalled: 'wait_capture_stalled',
   deadlineExceeded: 'wait_deadline_exceeded',
+  runnerRestartExhausted: 'wait_runner_restart_exhausted',
   targetAbsent: 'wait_target_absent',
   stableTimeout: 'wait_stable_timeout',
   landmarkIdentityMismatch: 'wait_landmark_identity_mismatch',

packages/platform-apple/src/runner/__tests__/runner-command-retry.test.ts

@@ -459,6 +459,49 @@ test('mutating commands emit readiness recovery diagnostics after failed preflig
   assert.match(diagnostics, /"recovery":"session_restarted"/);
 });
 
+test('mutating commands mark errors after a failed preflight runner restart', async () => {
+  const staleSession = makeRunnerSession({ port: 8100, ready: true });
+  const freshSession = makeRunnerSession({ port: 8101, ready: false });
+
+  mockEnsureRunnerSession.mockResolvedValueOnce(staleSession).mockResolvedValueOnce(freshSession);
+  mockExecuteRunnerCommandWithSession
+    .mockRejectedValueOnce(
+      new AppError('COMMAND_FAILED', 'fetch failed', {
+        runnerReadinessPreflightFailed: true,
+      }),
+    )
+    .mockRejectedValueOnce(
+      new AppError('COMMAND_FAILED', 'request canceled', {
+        diagnosticId: 'diag-restart',
+        logPath: '/tmp/restart.ndjson',
+      }),
+    );
+
+  await assert.rejects(
+    () =>
+      runAppleRunnerCommand(
+        IOS_SIMULATOR,
+        { command: 'tap', x: 120, y: 240 },
+        { logPath: '/tmp/runner.log' },
+      ),
+    (error: unknown) => {
+      assert.ok(error instanceof AppError);
+      assert.equal(error.details?.runnerRestarted, true);
+      assert.equal(
+        error.details?.runnerRestartReason,
+        'runner_readiness_preflight_failed_before_command_send',
+      );
+      assert.equal(error.details?.runnerRestartCommand, 'tap');
+      assert.match(String(error.details?.runnerRestartCommandId), /^runner-/);
+      assert.equal(error.details?.runnerInvalidatedSessionId, staleSession.sessionId);
+      assert.equal(error.details?.runnerRestartSessionId, freshSession.sessionId);
+      assert.equal(error.details?.diagnosticId, 'diag-restart');
+      assert.equal(error.details?.logPath, '/tmp/restart.ndjson');
+      return true;
+    },
+  );
+});
+
 test('mutating commands do not restart or replay after command send failure', async () => {
   const session = makeRunnerSession({ port: 8100, ready: true });
 

packages/platform-apple/src/runner/runner-lifecycle.ts

@@ -359,6 +359,8 @@ async function restartSessionAndRunCommand(params: {
   const restartedSession = await ensureRunnerSession(device, {
     ...options,
     cleanStaleBundles: true,
+  }).catch((error: unknown) => {
+    throw markRunnerRestartError(error, params);
   });
   commitRunnerRecycle(recycleKey);
   try {
@@ -386,21 +388,53 @@ async function restartSessionAndRunCommand(params: {
   } catch (retryErr) {
     const retryAppErr = asAppError(retryErr, 'COMMAND_FAILED');
     if (isRetryableRunnerError(retryAppErr)) {
-      return await handleRunnerTransportErrorAfterCommandSend({
-        device,
-        session: restartedSession,
-        command,
-        transportError: retryAppErr,
-        options,
-        signal,
-        invalidationReason: 'transport_error_after_retry_command_send',
-        invalidateSession: invalidateRunnerSession,
-      });
+      try {
+        return await handleRunnerTransportErrorAfterCommandSend({
+          device,
+          session: restartedSession,
+          command,
+          transportError: retryAppErr,
+          options,
+          signal,
+          invalidationReason: 'transport_error_after_retry_command_send',
+          invalidateSession: invalidateRunnerSession,
+        });
+      } catch (recoveryErr) {
+        throw markRunnerRestartError(recoveryErr, params, restartedSession);
+      }
     }
-    throw retryErr;
+    throw markRunnerRestartError(retryErr, params, restartedSession);
   }
 }
 
+function markRunnerRestartError(
+  error: unknown,
+  params: Pick<
+    Parameters<typeof restartSessionAndRunCommand>[0],
+    'session' | 'command' | 'options' | 'restartReason'
+  >,
+  restartedSession?: RunnerSession,
+): unknown {
+  if (!(error instanceof AppError)) return error;
+  return new AppError(
+    error.code,
+    error.message,
+    {
+      ...(error.details ?? {}),
+      runnerRestarted: true,
+      runnerRestartReason: params.restartReason,
+      runnerRestartCommand: params.command.command,
+      ...(params.command.commandId ? { runnerRestartCommandId: params.command.commandId } : {}),
+      runnerInvalidatedSessionId: params.session.sessionId,
+      ...(restartedSession ? { runnerRestartSessionId: restartedSession.sessionId } : {}),
+      ...(error.details?.logPath === undefined && params.options.logPath
+        ? { logPath: params.options.logPath }
+        : {}),
+    },
+    error.cause ?? error,
+  );
+}
+
 async function runPrepareHealthCheck(
   device: DeviceInfo,
   session: RunnerSession,

src/commands/interaction/runtime/wait-deadline.ts

@@ -1,4 +1,6 @@
-export type WaitDeadlineResult<T> = { timedOut: false; value: T } | { timedOut: true };
+export type WaitDeadlineResult<T> =
+  | { timedOut: false; value: T }
+  | { timedOut: true; error?: unknown };
 
 type AbortSignalSource = { signal?: AbortSignal };
 
@@ -37,7 +39,7 @@ export async function runWithinWaitDeadline<T>(
     return { timedOut: false, value };
   } catch (error) {
     if (deadlineExpired && !parentSignals.some((parent) => parent.aborted)) {
-      return { timedOut: true };
+      return { timedOut: true, error };
     }
     throw error;
   } finally {

src/commands/interaction/runtime/wait-polling.ts

@@ -1,4 +1,4 @@
-import { AppError } from '@agent-device/kernel/errors';
+import { AppError, type AppErrorDetails } from '@agent-device/kernel/errors';
 import { WAIT_REASONS } from '@agent-device/contracts/wait';
 import { isUnreadableCaptureContentError } from '@agent-device/contracts/android-snapshot-quality';
 import { selectorPollBudget } from '../../../core/selector-pipeline.ts';
@@ -15,12 +15,20 @@ import { runWithinWaitDeadline } from './wait-deadline.ts';
  */
 export const DEFAULT_WAIT_TIMEOUT_MS = SELECTOR_PIPELINE_POLICIES.wait.poll.defaultTimeoutMs;
 
-export type WaitPollDeadline = 'capture-stalled' | 'capture-truncated';
+export type WaitPollDeadline = 'capture-stalled' | 'capture-truncated' | 'runner-restart-exhausted';
 
 export type WaitFailureEvidence = {
   timeoutMs: number;
   readableCaptures: number;
   waitedMs: number;
+  runnerRestarted?: true;
+  runnerRestartReason?: string;
+  runnerRestartCommand?: string;
+  runnerRestartCommandId?: string;
+  runnerInvalidatedSessionId?: string;
+  runnerRestartSessionId?: string;
+  logPath?: string;
+  diagnosticId?: string;
 };
 
 type WaitPollingRuntime = {
@@ -62,6 +70,7 @@ export function createWaitPolling(
   const timeoutMs = requestedTimeoutMs ?? budget.defaultTimeoutMs;
   const startedAtMs = now(runtime);
   const unreadable = createUnreadablePollTracker();
+  let timeoutEvidence: Partial<WaitFailureEvidence> = {};
   const remainingMs = () => Math.max(0, timeoutMs - (now(runtime) - startedAtMs));
 
   return {
@@ -82,6 +91,8 @@ export function createWaitPolling(
         if (captureWasReadable) unreadable.recordReadableCapture();
         return result;
       }
+      const runnerRestart = runnerRestartTimeoutEvidence(result.error);
+      timeoutEvidence = runnerRestart ?? {};
       // A capture that only becomes readable after its deadline is not evidence for this wait.
       // Count only captures that completed before runWithinWaitDeadline returned a timeout.
       return {
@@ -90,16 +101,19 @@ export function createWaitPolling(
         // the poll index is not evidence. This remains true after one or more unreadable content
         // verdicts followed by a capture that consumes the remaining budget.
         deadline:
-          unreadable.readableCaptures() === 0
-            ? ('capture-stalled' as const)
-            : ('capture-truncated' as const),
+          runnerRestart !== undefined
+            ? ('runner-restart-exhausted' as const)
+            : unreadable.readableCaptures() === 0
+              ? ('capture-stalled' as const)
+              : ('capture-truncated' as const),
       };
     },
     hasTimeRemaining: () => remainingMs() > 0,
     failureEvidence: (): WaitFailureEvidence => ({
       timeoutMs,
       readableCaptures: unreadable.readableCaptures(),
       waitedMs: now(runtime) - startedAtMs,
+      ...timeoutEvidence,
     }),
     rethrowIfNeverReadable: unreadable.rethrowIfNeverReadable,
     sleepUntilNextPoll: async () =>
@@ -119,6 +133,16 @@ function waitCaptureStalledError(message: string, evidence: WaitFailureEvidence)
   });
 }
 
+function waitRunnerRestartExhaustedError(message: string, evidence: WaitFailureEvidence): AppError {
+  return new AppError('COMMAND_FAILED', message, {
+    reason: WAIT_REASONS.runnerRestartExhausted,
+    waitRunnerRestartExhausted: true,
+    ...evidence,
+    retriable: true,
+    hint: 'An iOS runner restart consumed the wait timeout before a readable snapshot completed. Inspect the diagnostics log for the runner invalidation/restart sequence, then retry.',
+  });
+}
+
 function waitDeadlineExceededError(message: string, evidence: WaitFailureEvidence): AppError {
   return new AppError('COMMAND_FAILED', message, {
     reason: WAIT_REASONS.deadlineExceeded,
@@ -140,6 +164,9 @@ export function waitTimeoutError(
   deadline: WaitPollDeadline | undefined,
 ): AppError {
   const evidence = polling.failureEvidence();
+  if (deadline === 'runner-restart-exhausted') {
+    return waitRunnerRestartExhaustedError(message, evidence);
+  }
   if (deadline === 'capture-stalled') return waitCaptureStalledError(message, evidence);
   if (deadline === 'capture-truncated') return waitDeadlineExceededError(message, evidence);
 
@@ -149,6 +176,30 @@ export function waitTimeoutError(
     : waitTargetAbsentError(message, evidence);
 }
 
+function runnerRestartTimeoutEvidence(error: unknown): Partial<WaitFailureEvidence> | undefined {
+  if (!(error instanceof AppError)) return undefined;
+  const details = error.details;
+  if (details?.runnerRestarted !== true) return undefined;
+  return {
+    runnerRestarted: true,
+    ...copyStringDetail(details, 'runnerRestartReason'),
+    ...copyStringDetail(details, 'runnerRestartCommand'),
+    ...copyStringDetail(details, 'runnerRestartCommandId'),
+    ...copyStringDetail(details, 'runnerInvalidatedSessionId'),
+    ...copyStringDetail(details, 'runnerRestartSessionId'),
+    ...copyStringDetail(details, 'logPath'),
+    ...copyStringDetail(details, 'diagnosticId'),
+  };
+}
+
+function copyStringDetail<Key extends keyof WaitFailureEvidence>(
+  details: AppErrorDetails | undefined,
+  key: Key,
+): Pick<WaitFailureEvidence, Key> | {} {
+  const value = details?.[key];
+  return typeof value === 'string' ? ({ [key]: value } as Pick<WaitFailureEvidence, Key>) : {};
+}
+
 function createUnreadablePollTracker(): UnreadablePollTracker {
   let readableCaptureCount = 0;
   let lastUnreadableError: unknown;

src/daemon/__tests__/wait-runtime.test.ts

@@ -1,5 +1,6 @@
 import { expect, test, vi } from 'vitest';
 import { WAIT_REASONS } from '@agent-device/contracts/wait';
+import { AppError } from '@agent-device/kernel/errors';
 import {
   type DeviceBinding,
   type RuntimeFacts,
@@ -532,6 +533,47 @@ test('a stalled capture reports capture-stalled with no readable captures', asyn
   expect(response.error.details?.readableCaptures).toBe(0);
 });
 
+test('a runner restart that exhausts the wait reports typed restart evidence', async () => {
+  const captureSnapshot = vi.fn(async (input: CaptureSnapshotInput) => {
+    const signal = input.signal;
+    if (!signal) throw new Error('the poll deadline never reached the platform');
+    await new Promise<void>((resolve) => {
+      if (signal.aborted) return resolve();
+      signal.addEventListener('abort', () => resolve(), { once: true });
+    });
+    throw new AppError('COMMAND_FAILED', 'request canceled', {
+      runnerRestarted: true,
+      runnerRestartReason: 'runner_readiness_preflight_failed_before_command_send',
+      runnerRestartCommand: 'snapshot',
+      runnerRestartCommandId: 'snapshot-1',
+      runnerInvalidatedSessionId: 'session-old',
+      runnerRestartSessionId: 'session-new',
+      diagnosticId: 'diag-restart',
+      logPath: '/tmp/restart.ndjson',
+    });
+  });
+  const harness = waitRuntimeHarness({ captureSnapshot });
+
+  const { response } = await runWait(['text', 'Ready', '50'], harness);
+
+  expect(response.ok).toBe(false);
+  if (response.ok) return;
+  expect(response.error.details?.reason).toBe(WAIT_REASONS.runnerRestartExhausted);
+  expect(response.error.details?.waitRunnerRestartExhausted).toBe(true);
+  expect(response.error.details?.captureStalled).toBeUndefined();
+  expect(response.error.details?.runnerRestarted).toBe(true);
+  expect(response.error.details?.runnerRestartReason).toBe(
+    'runner_readiness_preflight_failed_before_command_send',
+  );
+  expect(response.error.details?.runnerRestartCommand).toBe('snapshot');
+  expect(response.error.details?.runnerRestartCommandId).toBe('snapshot-1');
+  expect(response.error.details?.runnerInvalidatedSessionId).toBe('session-old');
+  expect(response.error.details?.runnerRestartSessionId).toBe('session-new');
+  expect(response.error.details?.diagnosticId).toBe('diag-restart');
+  expect(response.error.details?.logPath).toBe('/tmp/restart.ndjson');
+  expect(response.error.details?.readableCaptures).toBe(0);
+});
+
 test('a readable capture that lacks the target stays target-absent, not capture-stalled', async () => {
   const harness = waitRuntimeHarness({
     nodesPerPoll: [[{ index: 0, depth: 0, type: 'Button', label: 'Checkout', hittable: true }]],