feat: verify device cloud connections (#1564)

* feat: verify device cloud connections

* refactor: unify connect provider adapters

* refactor: separate connect verification facts

* fix: tighten connect provider verification

* fix: use neutral cloud connection wording

* perf: deduplicate local affected checks

* refactor: simplify affected check runner

* refactor: derive connect workflow from verification

Author: thymikee

docs/agents/testing.md

@@ -141,6 +141,12 @@ docs-only short-circuit its path would otherwise take. If the matrix moves
 again, move that entry with it.
 The plan documents the rule and changed path behind every selected check.
 
+Local coverage reuses the affected Vitest run as its LCOV producer and applies the changed-line
+coverage gate to that report. It does not run the full instrumented suite; global coverage
+thresholds and full unit/provider matrices remain authoritative in GitHub CI. When coverage is
+selected, `vitest-related` is folded into this one affected coverage run, and full unit/provider
+aggregates are not repeated locally.
+
 Model and catalog live under `scripts/check-affected/`; the derivation is guarded
 by `pnpm check:affected:test` (the `Affected-check Selector` CI job).
 

packages/contracts/src/facades/remote.ts

@@ -1,3 +1,4 @@
 export * from '../companion-tunnel-scope.ts';
 export * from '../metro.ts';
 export * from '../remote-config-fields.ts';
+export * from '../provider-connection.ts';

packages/contracts/src/provider-connection.ts

@@ -0,0 +1,19 @@
+export type ProviderConnectionResource = {
+  status: 'verified' | 'configured' | 'deferred' | 'missing';
+  name?: string;
+  reference?: string;
+  platform?: 'android' | 'ios';
+  osVersion?: string;
+  version?: string;
+  availability?: string;
+  message?: string;
+};
+
+export type ProviderConnectionVerification = {
+  provider: string;
+  service: string;
+  verificationMessage: string;
+  project?: { name?: string; reference: string };
+  device: ProviderConnectionResource;
+  app: ProviderConnectionResource;
+};

packages/provider-limrun/src/client-options.ts

@@ -0,0 +1,14 @@
+export const LIMRUN_CLIENT_HEADER = 'agent-device-cli';
+
+export function buildLimrunClientOptions(options: { apiKey: string; clientVersion: string }): {
+  apiKey: string;
+  defaultHeaders: Record<string, string>;
+} {
+  return {
+    apiKey: options.apiKey,
+    defaultHeaders: {
+      'x-agent-device-client': LIMRUN_CLIENT_HEADER,
+      'x-agent-device-version': options.clientVersion,
+    },
+  };
+}

packages/provider-limrun/src/connection-verification.test.ts

@@ -0,0 +1,72 @@
+import assert from 'node:assert/strict';
+import { afterEach, test, vi } from 'vitest';
+import { verifyLimrunConnection } from './connection-verification.ts';
+
+const mockState = vi.hoisted(() => ({
+  constructorOptions: [] as Array<Record<string, unknown>>,
+  androidList: vi.fn(async () => ({ getPaginatedItems: () => [] })),
+  iosList: vi.fn(async () => ({ getPaginatedItems: () => [] })),
+}));
+
+vi.mock('@limrun/api', () => ({
+  default: class MockLimrun {
+    readonly androidInstances = { list: mockState.androidList };
+    readonly iosInstances = { list: mockState.iosList };
+
+    constructor(options: Record<string, unknown>) {
+      mockState.constructorOptions.push(options);
+    }
+  },
+}));
+
+afterEach(() => {
+  mockState.constructorOptions.length = 0;
+  vi.clearAllMocks();
+});
+
+test('Limrun verification reads the selected instance service without creating an instance', async () => {
+  const result = await verifyLimrunConnection({
+    apiKey: 'lim_test_key',
+    clientVersion: '1.2.3',
+    platform: 'android',
+  });
+
+  assert.deepEqual(result, {
+    provider: 'limrun',
+    service: 'Limrun',
+    verificationMessage: 'Credentials and Android instance access verified.',
+    device: {
+      status: 'deferred',
+      name: 'Provider-selected Android emulator',
+      platform: 'android',
+    },
+    app: {
+      status: 'missing',
+      message: 'A new Limrun instance does not have your app yet.',
+    },
+  });
+  assert.deepEqual(mockState.androidList.mock.calls, [[{ limit: 1 }]]);
+  assert.equal(mockState.iosList.mock.calls.length, 0);
+  assert.equal(mockState.constructorOptions[0]?.apiKey, 'lim_test_key');
+  assert.deepEqual(mockState.constructorOptions[0]?.defaultHeaders, {
+    'x-agent-device-client': 'agent-device-cli',
+    'x-agent-device-version': '1.2.3',
+  });
+});
+
+test('Limrun verification classifies authentication failures', async () => {
+  mockState.iosList.mockRejectedValueOnce(Object.assign(new Error('invalid'), { status: 401 }));
+
+  await assert.rejects(
+    verifyLimrunConnection({
+      apiKey: 'lim_bad_key',
+      clientVersion: '1.2.3',
+      platform: 'ios',
+    }),
+    (error: unknown) => {
+      assert.equal((error as { code?: string }).code, 'UNAUTHORIZED');
+      assert.doesNotMatch(JSON.stringify(error), /lim_bad_key/);
+      return true;
+    },
+  );
+});

packages/provider-limrun/src/connection-verification.ts

@@ -0,0 +1,79 @@
+import Limrun from '@limrun/api';
+import { AppError } from '@agent-device/kernel/errors';
+import { buildLimrunClientOptions } from './client-options.ts';
+import type { ProviderConnectionVerification } from '@agent-device/contracts/remote';
+
+export type LimrunConnectionVerification = ProviderConnectionVerification & {
+  provider: 'limrun';
+  service: 'Limrun';
+  device: {
+    status: 'deferred';
+    name: string;
+    platform: 'android' | 'ios';
+  };
+  app: {
+    status: 'missing';
+    message: string;
+  };
+};
+
+export type LimrunConnectionVerificationOptions = {
+  apiKey: string;
+  clientVersion: string;
+  platform: 'android' | 'ios';
+  region?: string;
+};
+
+export async function verifyLimrunConnection(
+  options: LimrunConnectionVerificationOptions,
+): Promise<LimrunConnectionVerification> {
+  const client = new Limrun({
+    ...buildLimrunClientOptions(options),
+    timeout: 15_000,
+    maxRetries: 0,
+  });
+  const query = { limit: 1, ...(options.region ? { region: options.region } : {}) };
+  try {
+    if (options.platform === 'android') {
+      await client.androidInstances.list(query);
+    } else {
+      await client.iosInstances.list(query);
+    }
+  } catch (error) {
+    const status = readStatus(error);
+    if (status === 401 || status === 403) {
+      throw new AppError('UNAUTHORIZED', 'Limrun rejected connection verification.', {
+        status,
+        hint: 'Check LIMRUN_API_KEY and its organization access.',
+      });
+    }
+    throw new AppError(
+      'COMMAND_FAILED',
+      'Limrun connection verification failed.',
+      { hint: 'Check Limrun service access, LIMRUN_REGION, and network connectivity, then retry.' },
+      error,
+    );
+  }
+  const platformName = options.platform === 'android' ? 'Android' : 'iOS';
+  const deviceKind = options.platform === 'android' ? 'emulator' : 'simulator';
+  return {
+    provider: 'limrun',
+    service: 'Limrun',
+    verificationMessage: `Credentials and ${platformName} instance access verified.`,
+    device: {
+      status: 'deferred',
+      name: `Provider-selected ${platformName} ${deviceKind}`,
+      platform: options.platform,
+    },
+    app: {
+      status: 'missing',
+      message: 'A new Limrun instance does not have your app yet.',
+    },
+  };
+}
+
+function readStatus(error: unknown): number | undefined {
+  if (!error || typeof error !== 'object') return undefined;
+  const status = (error as { status?: unknown }).status;
+  return typeof status === 'number' ? status : undefined;
+}

packages/provider-limrun/src/index.ts

@@ -1,5 +1,10 @@
 export { LIMRUN_PROVIDER } from './device.ts';
 export { createLimrunRuntime, type LimrunRuntime, type LimrunRuntimeOptions } from './runtime.ts';
+export {
+  verifyLimrunConnection,
+  type LimrunConnectionVerification,
+  type LimrunConnectionVerificationOptions,
+} from './connection-verification.ts';
 
 export type { LimrunRuntimeDependencies } from './runtime-dependencies.ts';
 

packages/provider-limrun/src/runtime.ts

@@ -34,6 +34,7 @@ import {
 } from './ios.ts';
 import { createLimrunDeviceSession, type LimrunDeviceSession } from './device-session.ts';
 import type { LimrunRuntimeDependencies } from './runtime-dependencies.ts';
+import { buildLimrunClientOptions, LIMRUN_CLIENT_HEADER } from './client-options.ts';
 
 type LimrunInstance = {
   metadata: { id: string };
@@ -51,8 +52,6 @@ export type LimrunRuntimeOptions = {
   region?: string;
 };
 
-const LIMRUN_CLIENT_HEADER = 'agent-device-cli';
-
 export type LimrunRuntime = ProviderDeviceRuntime & {
   recoverExpiredLease: ProviderExpiredLeaseRecovery;
   getDeviceSession(device: DeviceInfo): LimrunDeviceSession | undefined;
@@ -99,13 +98,12 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime {
   constructor(options: LimrunRuntimeOptions, dependencies: LimrunRuntimeDependencies) {
     this.options = options;
     this.dependencies = dependencies;
-    this.limrun = new Limrun({
-      apiKey: options.apiKey,
-      defaultHeaders: {
-        'x-agent-device-client': LIMRUN_CLIENT_HEADER,
-        'x-agent-device-version': dependencies.clientVersion,
-      },
-    });
+    this.limrun = new Limrun(
+      buildLimrunClientOptions({
+        apiKey: options.apiKey,
+        clientVersion: dependencies.clientVersion,
+      }),
+    );
   }
 
   ownsDevice(device: DeviceInfo): boolean {

packages/provider-webdriver/src/aws-device-farm-connection-verification.ts

@@ -0,0 +1,120 @@
+import { AppError } from '@agent-device/kernel/errors';
+import { createAwsDeviceFarmCommandRunner } from './aws-device-farm.ts';
+import type { RunHostCommand } from './dependencies.ts';
+import type {
+  CloudWebDriverConnectionVerification,
+  CloudWebDriverConnectionVerificationOptions,
+} from './connection-verification.ts';
+import type { ProviderConnectionResource } from '@agent-device/contracts/remote';
+
+type AwsOptions = Extract<
+  CloudWebDriverConnectionVerificationOptions,
+  { provider: 'aws-device-farm' }
+>;
+
+export function readAwsDeviceFarmRegionFromArn(arn: string): string | undefined {
+  return /^arn:[^:]+:devicefarm:([^:]+):/.exec(arn)?.[1];
+}
+
+export async function verifyAwsDeviceFarmConnection(
+  options: AwsOptions,
+  runHostCommand: RunHostCommand,
+): Promise<CloudWebDriverConnectionVerification> {
+  const runAwsJson = createAwsDeviceFarmCommandRunner({
+    runHostCommand,
+    region: options.region,
+  });
+  const [projectResponse, deviceResponse, uploadResponse] = await Promise.all([
+    runAwsJson('get-project', ['--arn', options.projectArn]),
+    runAwsJson('get-device', ['--arn', options.deviceArn]),
+    options.appArn ? runAwsJson('get-upload', ['--arn', options.appArn]) : undefined,
+  ]);
+  const project = readAwsResource(projectResponse, 'project');
+  const device = readAwsResource(deviceResponse, 'device');
+  const devicePlatform = readAwsPlatform(device.platform);
+  if (devicePlatform !== options.platform) {
+    throw new AppError(
+      'INVALID_ARGS',
+      `AWS Device Farm device "${readString(device.name) ?? options.deviceArn}" is ${devicePlatform}, not ${options.platform}.`,
+    );
+  }
+
+  const app = options.appArn
+    ? verifyAwsUpload(readAwsResource(uploadResponse, 'upload'), options)
+    : {
+        status: 'missing' as const,
+        message:
+          'No app upload is attached; AWS Device Farm does not support install after allocation. Reconnect with --aws-app-arn <arn>.',
+      };
+  return {
+    provider: 'aws-device-farm',
+    service: 'AWS Device Farm',
+    verificationMessage: options.appArn
+      ? 'Credentials, project, device, and app upload verified.'
+      : 'Credentials, project, and device verified.',
+    project: { name: readString(project.name), reference: options.projectArn },
+    device: {
+      status: 'verified',
+      name: readString(device.name) ?? options.deviceArn,
+      reference: options.deviceArn,
+      platform: devicePlatform,
+      osVersion: readString(device.os),
+      availability: readString(device.availability),
+    },
+    app,
+  };
+}
+
+function verifyAwsUpload(
+  upload: Record<string, unknown>,
+  options: AwsOptions,
+): ProviderConnectionResource {
+  const status = readString(upload.status);
+  if (status !== 'SUCCEEDED') {
+    throw new AppError(
+      'COMMAND_FAILED',
+      `AWS Device Farm app upload is not ready (${status ?? 'unknown status'}).`,
+      {
+        status,
+        hint: 'Wait for the upload to succeed or reconnect with a different --aws-app-arn.',
+      },
+    );
+  }
+  const expectedType = options.platform === 'android' ? 'ANDROID_APP' : 'IOS_APP';
+  const type = readString(upload.type);
+  if (type !== expectedType) {
+    throw new AppError(
+      'INVALID_ARGS',
+      `AWS Device Farm app upload type is ${type ?? 'unknown'}, expected ${expectedType}.`,
+    );
+  }
+  return {
+    status: 'verified',
+    name: readString(upload.name),
+    reference: options.appArn,
+  };
+}
+
+function readAwsResource(value: unknown, key: string): Record<string, unknown> {
+  if (!value || typeof value !== 'object') {
+    throw new AppError('COMMAND_FAILED', `AWS Device Farm ${key} response was not an object.`);
+  }
+  const resource = (value as Record<string, unknown>)[key];
+  if (!resource || typeof resource !== 'object' || Array.isArray(resource)) {
+    throw new AppError('COMMAND_FAILED', `AWS Device Farm response missed ${key}.`);
+  }
+  return resource as Record<string, unknown>;
+}
+
+function readAwsPlatform(value: unknown): 'android' | 'ios' {
+  if (value === 'ANDROID') return 'android';
+  if (value === 'IOS') return 'ios';
+  throw new AppError(
+    'UNSUPPORTED_PLATFORM',
+    `AWS Device Farm device platform is ${String(value)}.`,
+  );
+}
+
+function readString(value: unknown): string | undefined {
+  return typeof value === 'string' && value.length > 0 ? value : undefined;
+}

packages/provider-webdriver/src/aws-device-farm.ts

@@ -307,10 +307,19 @@ async function runAwsJson(
   args: string[],
 ): Promise<unknown> {
   const result = await runHostCommand(command, args);
-  return JSON.parse(result.stdout) as unknown;
+  try {
+    return JSON.parse(result.stdout) as unknown;
+  } catch (error) {
+    throw new AppError(
+      'COMMAND_FAILED',
+      'AWS Device Farm returned invalid JSON.',
+      undefined,
+      error,
+    );
+  }
 }
 
-function createAwsDeviceFarmCommandRunner(
+export function createAwsDeviceFarmCommandRunner(
   options: AwsCliDeviceFarmClientOptions,
 ): (subcommand: string, args: string[]) => Promise<unknown> {
   const regionArgs = options.region ? ['--region', options.region] : [];