|
| 1 | +import os from 'node:os'; |
| 2 | +import path from 'node:path'; |
| 3 | +import { expect, test, vi } from 'vitest'; |
| 4 | +import type { SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; |
| 5 | +import { makeIosSession } from '../../__tests__/test-utils/session-factories.ts'; |
| 6 | +import { |
| 7 | + recoveredSnapshotQualityWarning, |
| 8 | + renderSnapshotQualityWarnings, |
| 9 | +} from '../../snapshot/snapshot-quality.ts'; |
| 10 | +import { |
| 11 | + applyRecoveredWarningLatch, |
| 12 | + resolveRecoveredWarningLatch, |
| 13 | +} from '../snapshot-quality-latch.ts'; |
| 14 | +import { dispatchSnapshotDiffViaRuntime, dispatchSnapshotViaRuntime } from '../snapshot-runtime.ts'; |
| 15 | +import { SessionStore } from '../session-store.ts'; |
| 16 | +import type { SessionState } from '../types.ts'; |
| 17 | + |
| 18 | +const dispatchCommandMock = vi.hoisted(() => vi.fn()); |
| 19 | + |
| 20 | +vi.mock('../../core/dispatch.ts', async (importOriginal) => { |
| 21 | + const actual = await importOriginal<typeof import('../../core/dispatch.ts')>(); |
| 22 | + return { |
| 23 | + ...actual, |
| 24 | + dispatchCommand: dispatchCommandMock, |
| 25 | + }; |
| 26 | +}); |
| 27 | + |
| 28 | +const FULL_WARNING = recoveredSnapshotQualityWarning('private-ax'); |
| 29 | + |
| 30 | +function deferredVerdict(): SnapshotQualityVerdict { |
| 31 | + return { |
| 32 | + state: 'recovered', |
| 33 | + backend: 'private-ax', |
| 34 | + reason: 'XCTest-backed snapshot tiers were deferred after recent slow accessibility work', |
| 35 | + reasonCode: 'deferred', |
| 36 | + }; |
| 37 | +} |
| 38 | + |
| 39 | +function genuineRecoveredVerdict(): SnapshotQualityVerdict { |
| 40 | + return { |
| 41 | + state: 'recovered', |
| 42 | + backend: 'private-ax', |
| 43 | + reason: 'iOS XCTest snapshot failed while serializing the accessibility tree', |
| 44 | + reasonCode: 'ax-rejected', |
| 45 | + }; |
| 46 | +} |
| 47 | + |
| 48 | +test('the injected warning is the exact line a genuine recovery renders', () => { |
| 49 | + // The latch exists to restore the warning an internal arming capture never rendered; |
| 50 | + // wording drift between the two paths would defeat one-shot deduplication. |
| 51 | + expect(renderSnapshotQualityWarnings(genuineRecoveredVerdict(), [])).toEqual([FULL_WARNING]); |
| 52 | +}); |
| 53 | + |
| 54 | +test('resolveRecoveredWarningLatch transitions', () => { |
| 55 | + const app = 'com.example.app'; |
| 56 | + |
| 57 | + // First deferred verdict with no latch held: inject once and hold the latch. |
| 58 | + const first = resolveRecoveredWarningLatch({ |
| 59 | + verdict: deferredVerdict(), |
| 60 | + appBundleId: app, |
| 61 | + latch: undefined, |
| 62 | + }); |
| 63 | + expect(first.warning).toBe(FULL_WARNING); |
| 64 | + expect(first.latch).toEqual({ appBundleId: app }); |
| 65 | + |
| 66 | + // Held latch: further deferred verdicts stay quiet. |
| 67 | + const repeat = resolveRecoveredWarningLatch({ |
| 68 | + verdict: deferredVerdict(), |
| 69 | + appBundleId: app, |
| 70 | + latch: first.latch, |
| 71 | + }); |
| 72 | + expect(repeat.warning).toBeUndefined(); |
| 73 | + expect(repeat.latch).toEqual({ appBundleId: app }); |
| 74 | + |
| 75 | + // A genuine recovery renders through the runtime, so it sets the latch silently. |
| 76 | + const genuine = resolveRecoveredWarningLatch({ |
| 77 | + verdict: genuineRecoveredVerdict(), |
| 78 | + appBundleId: app, |
| 79 | + latch: undefined, |
| 80 | + }); |
| 81 | + expect(genuine.warning).toBeUndefined(); |
| 82 | + expect(genuine.latch).toEqual({ appBundleId: app }); |
| 83 | + |
| 84 | + // Healthy ends the penalty window; the next arming warns again. |
| 85 | + const healthy = resolveRecoveredWarningLatch({ |
| 86 | + verdict: { state: 'healthy', backend: 'tree' }, |
| 87 | + appBundleId: app, |
| 88 | + latch: first.latch, |
| 89 | + }); |
| 90 | + expect(healthy.latch).toBeUndefined(); |
| 91 | + |
| 92 | + // Sparse and verdict-less captures carry no penalty signal either way. |
| 93 | + const sparse = resolveRecoveredWarningLatch({ |
| 94 | + verdict: { state: 'sparse', backend: 'private-ax' }, |
| 95 | + appBundleId: app, |
| 96 | + latch: first.latch, |
| 97 | + }); |
| 98 | + expect(sparse.warning).toBeUndefined(); |
| 99 | + expect(sparse.latch).toEqual({ appBundleId: app }); |
| 100 | + const absent = resolveRecoveredWarningLatch({ |
| 101 | + verdict: undefined, |
| 102 | + appBundleId: app, |
| 103 | + latch: first.latch, |
| 104 | + }); |
| 105 | + expect(absent.warning).toBeUndefined(); |
| 106 | + expect(absent.latch).toEqual({ appBundleId: app }); |
| 107 | + |
| 108 | + // The runner clears its penalty on app switch, so a latch held for another |
| 109 | + // app never suppresses the new app's first deferred verdict. |
| 110 | + const switched = resolveRecoveredWarningLatch({ |
| 111 | + verdict: deferredVerdict(), |
| 112 | + appBundleId: 'com.example.other', |
| 113 | + latch: first.latch, |
| 114 | + }); |
| 115 | + expect(switched.warning).toBe(FULL_WARNING); |
| 116 | + expect(switched.latch).toEqual({ appBundleId: 'com.example.other' }); |
| 117 | +}); |
| 118 | + |
| 119 | +test('internal observation responses neither consume nor clear the latch', () => { |
| 120 | + const session = makeIosSession('default', { appBundleId: 'com.example.app' }); |
| 121 | + |
| 122 | + const internal = applyRecoveredWarningLatch({ |
| 123 | + session, |
| 124 | + data: {}, |
| 125 | + verdict: deferredVerdict(), |
| 126 | + internalObservation: true, |
| 127 | + }); |
| 128 | + expect(internal.warnings).toBeUndefined(); |
| 129 | + expect(session.recoveredSnapshotWarningLatch).toBeUndefined(); |
| 130 | + |
| 131 | + session.recoveredSnapshotWarningLatch = { appBundleId: 'com.example.app' }; |
| 132 | + applyRecoveredWarningLatch({ |
| 133 | + session, |
| 134 | + data: {}, |
| 135 | + verdict: { state: 'healthy', backend: 'tree' }, |
| 136 | + internalObservation: true, |
| 137 | + }); |
| 138 | + expect(session.recoveredSnapshotWarningLatch).toEqual({ appBundleId: 'com.example.app' }); |
| 139 | +}); |
| 140 | + |
| 141 | +test('sessionless responses pass through unchanged', () => { |
| 142 | + const data = {}; |
| 143 | + expect( |
| 144 | + applyRecoveredWarningLatch({ |
| 145 | + session: undefined, |
| 146 | + data, |
| 147 | + verdict: deferredVerdict(), |
| 148 | + internalObservation: false, |
| 149 | + }), |
| 150 | + ).toBe(data); |
| 151 | +}); |
| 152 | + |
| 153 | +function scenario() { |
| 154 | + const root = path.join(os.tmpdir(), `agent-device-quality-latch-${crypto.randomUUID()}`); |
| 155 | + const sessionStore = new SessionStore(path.join(root, 'sessions')); |
| 156 | + const sessionName = 'default'; |
| 157 | + const session = makeIosSession(sessionName, { appBundleId: 'com.example.app' }); |
| 158 | + sessionStore.set(sessionName, session); |
| 159 | + return { sessionStore, sessionName, logPath: path.join(root, 'daemon.log') }; |
| 160 | +} |
| 161 | + |
| 162 | +function seedCapture(verdict: SnapshotQualityVerdict, label = 'Continue') { |
| 163 | + dispatchCommandMock.mockResolvedValue({ |
| 164 | + backend: 'xctest', |
| 165 | + truncated: false, |
| 166 | + quality: verdict, |
| 167 | + nodes: [ |
| 168 | + { |
| 169 | + index: 0, |
| 170 | + depth: 0, |
| 171 | + type: 'Button', |
| 172 | + label, |
| 173 | + rect: { x: 0, y: 0, width: 100, height: 44 }, |
| 174 | + hittable: true, |
| 175 | + }, |
| 176 | + ], |
| 177 | + }); |
| 178 | +} |
| 179 | + |
| 180 | +async function dispatchPublicSnapshot(input: ReturnType<typeof scenario>) { |
| 181 | + return await dispatchSnapshotViaRuntime({ |
| 182 | + req: { command: 'snapshot', positionals: [], token: 't', session: input.sessionName }, |
| 183 | + sessionName: input.sessionName, |
| 184 | + logPath: input.logPath, |
| 185 | + sessionStore: input.sessionStore, |
| 186 | + }); |
| 187 | +} |
| 188 | + |
| 189 | +function responseWarnings(response: Awaited<ReturnType<typeof dispatchSnapshotViaRuntime>>) { |
| 190 | + if (!response.ok) throw new Error('expected ok response'); |
| 191 | + return (response.data?.warnings ?? []) as string[]; |
| 192 | +} |
| 193 | + |
| 194 | +function storedLatch( |
| 195 | + input: ReturnType<typeof scenario>, |
| 196 | +): SessionState['recoveredSnapshotWarningLatch'] { |
| 197 | + return input.sessionStore.get(input.sessionName)?.recoveredSnapshotWarningLatch; |
| 198 | +} |
| 199 | + |
| 200 | +test('an internally armed penalty warns once on the first public deferred snapshot', async () => { |
| 201 | + const input = scenario(); |
| 202 | + seedCapture(deferredVerdict()); |
| 203 | + |
| 204 | + // The internal capture that armed the penalty (settle observation, selector |
| 205 | + // resolution) rendered nothing; without the latch the deferred suppression |
| 206 | + // would hide the warning from the user entirely. |
| 207 | + const internal = await dispatchSnapshotViaRuntime({ |
| 208 | + req: { |
| 209 | + command: 'snapshot', |
| 210 | + positionals: [], |
| 211 | + token: 't', |
| 212 | + session: input.sessionName, |
| 213 | + internal: { observationOnly: true }, |
| 214 | + }, |
| 215 | + sessionName: input.sessionName, |
| 216 | + logPath: input.logPath, |
| 217 | + sessionStore: input.sessionStore, |
| 218 | + }); |
| 219 | + expect(responseWarnings(internal)).not.toContain(FULL_WARNING); |
| 220 | + expect(storedLatch(input)).toBeUndefined(); |
| 221 | + |
| 222 | + const first = await dispatchPublicSnapshot(input); |
| 223 | + expect(responseWarnings(first).filter((line) => line === FULL_WARNING)).toHaveLength(1); |
| 224 | + expect(storedLatch(input)).toEqual({ appBundleId: 'com.example.app' }); |
| 225 | + |
| 226 | + const second = await dispatchPublicSnapshot(input); |
| 227 | + expect(responseWarnings(second)).not.toContain(FULL_WARNING); |
| 228 | +}); |
| 229 | + |
| 230 | +test('a healthy public capture re-arms the one-shot warning', async () => { |
| 231 | + const input = scenario(); |
| 232 | + seedCapture(deferredVerdict()); |
| 233 | + await dispatchPublicSnapshot(input); |
| 234 | + expect(storedLatch(input)).toEqual({ appBundleId: 'com.example.app' }); |
| 235 | + |
| 236 | + seedCapture({ state: 'healthy', backend: 'tree' }); |
| 237 | + const healthy = await dispatchPublicSnapshot(input); |
| 238 | + expect(responseWarnings(healthy)).not.toContain(FULL_WARNING); |
| 239 | + expect(storedLatch(input)).toBeUndefined(); |
| 240 | + |
| 241 | + seedCapture(deferredVerdict()); |
| 242 | + const rearmed = await dispatchPublicSnapshot(input); |
| 243 | + expect(responseWarnings(rearmed).filter((line) => line === FULL_WARNING)).toHaveLength(1); |
| 244 | +}); |
| 245 | + |
| 246 | +test('an empty ref-scoped diff latches on the captured verdict, not the retained snapshot', async () => { |
| 247 | + const input = scenario(); |
| 248 | + // The stored snapshot is healthy and carries the ref the diff will scope to; |
| 249 | + // the fresh capture is deferred and contains no node matching that scope, so |
| 250 | + // the empty scoped result deliberately retains the stored snapshot |
| 251 | + // (`shouldKeepCurrentSnapshot`) — a seam reading the verdict back from the |
| 252 | + // session would consult the retained healthy verdict and omit the warning. |
| 253 | + const session = input.sessionStore.get(input.sessionName)!; |
| 254 | + session.snapshot = { |
| 255 | + createdAt: Date.now(), |
| 256 | + snapshotQuality: { state: 'healthy', backend: 'tree' }, |
| 257 | + nodes: [ |
| 258 | + { |
| 259 | + index: 0, |
| 260 | + depth: 0, |
| 261 | + type: 'Button', |
| 262 | + ref: 'e1', |
| 263 | + label: 'Continue', |
| 264 | + rect: { x: 0, y: 0, width: 100, height: 44 }, |
| 265 | + hittable: true, |
| 266 | + }, |
| 267 | + ], |
| 268 | + }; |
| 269 | + // The deferred capture holds no node labeled 'Continue', so the '@e1' scope |
| 270 | + // resolves to zero nodes and the retention path runs. |
| 271 | + seedCapture(deferredVerdict(), 'Something else'); |
| 272 | + |
| 273 | + const diff = await dispatchSnapshotDiffViaRuntime({ |
| 274 | + req: { |
| 275 | + command: 'diff', |
| 276 | + positionals: [], |
| 277 | + token: 't', |
| 278 | + session: input.sessionName, |
| 279 | + flags: { snapshotScope: '@e1' }, |
| 280 | + }, |
| 281 | + sessionName: input.sessionName, |
| 282 | + logPath: input.logPath, |
| 283 | + sessionStore: input.sessionStore, |
| 284 | + }); |
| 285 | + |
| 286 | + // The retention actually happened: the stored snapshot (and its healthy |
| 287 | + // verdict) survived the empty scoped capture. |
| 288 | + const retained = input.sessionStore.get(input.sessionName)?.snapshot; |
| 289 | + expect(retained?.nodes[0]?.label).toBe('Continue'); |
| 290 | + expect(retained?.snapshotQuality?.state).toBe('healthy'); |
| 291 | + |
| 292 | + expect(responseWarnings(diff).filter((line) => line === FULL_WARNING)).toHaveLength(1); |
| 293 | + expect(storedLatch(input)).toEqual({ appBundleId: 'com.example.app' }); |
| 294 | +}); |
| 295 | + |
| 296 | +test('an app switch supersedes the held latch through the dispatch seam', async () => { |
| 297 | + const input = scenario(); |
| 298 | + seedCapture(deferredVerdict()); |
| 299 | + await dispatchPublicSnapshot(input); |
| 300 | + expect(storedLatch(input)).toEqual({ appBundleId: 'com.example.app' }); |
| 301 | + |
| 302 | + // The runner clears its penalty per target process, so a latch held for the |
| 303 | + // previous app must not silence the new app's first deferred verdict. |
| 304 | + input.sessionStore.get(input.sessionName)!.appBundleId = 'com.example.other'; |
| 305 | + const switched = await dispatchPublicSnapshot(input); |
| 306 | + expect(responseWarnings(switched).filter((line) => line === FULL_WARNING)).toHaveLength(1); |
| 307 | + expect(storedLatch(input)).toEqual({ appBundleId: 'com.example.other' }); |
| 308 | +}); |
| 309 | + |
| 310 | +test('a genuine recovered render keeps later deferred captures quiet without doubling', async () => { |
| 311 | + const input = scenario(); |
| 312 | + seedCapture(genuineRecoveredVerdict()); |
| 313 | + |
| 314 | + const genuine = await dispatchPublicSnapshot(input); |
| 315 | + expect(responseWarnings(genuine).filter((line) => line === FULL_WARNING)).toHaveLength(1); |
| 316 | + expect(storedLatch(input)).toEqual({ appBundleId: 'com.example.app' }); |
| 317 | + |
| 318 | + seedCapture(deferredVerdict()); |
| 319 | + const deferred = await dispatchPublicSnapshot(input); |
| 320 | + expect(responseWarnings(deferred)).not.toContain(FULL_WARNING); |
| 321 | +}); |
0 commit comments