Skip to content

Commit 3e584d1

Browse files
authored
fix(ios): isolate snapshot acquisition timing from presentation (#1948)
* fix(ios): isolate snapshot acquisition timing * fix(ios): pin snapshot phase timing fixture * fix(ios): avoid starving synthesized text commits
1 parent d2f2dae commit 3e584d1

7 files changed

Lines changed: 370 additions & 111 deletions

File tree

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift

Lines changed: 148 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ struct SnapshotQuality: Codable {
2525
let collapsedLeafIndexes: [Int]?
2626
/// Coverage of the bounded custom-action pass, when the capture asked for one.
2727
let customActions: SnapshotCustomActionCoverage?
28+
/// Response-level timing for the accepted backend attempt, never repeated per node.
29+
var timing: SnapshotCaptureTiming? = nil
2830
}
2931

3032
/// How much of the merged-element set the custom-action pass actually read. An
@@ -76,6 +78,9 @@ struct SnapshotBackendCapture {
7678
/// Broad presentation used only by the quality classifier when a scope narrows publication.
7779
/// A legitimate missing scope is an empty healthy projection, not backend failure evidence.
7880
var qualityPayload: DataPayload? = nil
81+
/// Set by the capture plan after measuring acquisition and presentation separately. Direct
82+
/// presentation fixtures do not claim a plan timing.
83+
var timing: SnapshotCaptureTiming? = nil
7984
}
8085

8186
extension RunnerTests {
@@ -327,42 +332,19 @@ extension RunnerTests {
327332
}
328333
continue
329334
}
330-
let capture: SnapshotBackendCapture
331-
let backendStartedAt = Date()
332-
do {
333-
guard
334-
let result = try captureWithBackend(
335-
kind,
336-
app: app,
337-
options: options,
338-
deadline: deadline,
339-
treeCaptureSliceBudgetOverride: effective.treeCaptureSliceBudgetOverride
340-
)
341-
else {
342-
recordSlowXCTestSnapshotBackendIfNeeded(
343-
kind,
344-
startedAt: backendStartedAt,
345-
penaltySuppressed: suppressXCTestPenalty
346-
)
347-
continue
348-
}
349-
capture = result
350-
recordSlowXCTestSnapshotBackendIfNeeded(
351-
kind,
352-
startedAt: backendStartedAt,
353-
penaltySuppressed: suppressXCTestPenalty
354-
)
355-
} catch let failure as SnapshotCaptureFailure {
356-
recordXCTestSnapshotBackendFailureIfNeeded(
357-
kind,
358-
failure: failure,
359-
penaltySuppressed: suppressXCTestPenalty
360-
)
361-
recordSlowXCTestSnapshotBackendIfNeeded(
362-
kind,
363-
startedAt: backendStartedAt,
364-
penaltySuppressed: suppressXCTestPenalty
365-
)
335+
let attempt = try captureWithBackend(
336+
kind,
337+
app: app,
338+
options: options,
339+
deadline: deadline,
340+
treeCaptureSliceBudgetOverride: effective.treeCaptureSliceBudgetOverride
341+
)
342+
recordXCTestSnapshotBackendAttemptIfNeeded(
343+
kind,
344+
attempt: attempt,
345+
penaltySuppressed: suppressXCTestPenalty
346+
)
347+
if case let .failed(failure, phase: _) = attempt.outcome {
366348
if Self.isAxSnapshotFailure(failure) { axFailure = failure }
367349
if firstFailure == nil {
368350
firstFailure = (failure.message, Self.isAxSnapshotFailure(failure) ? "ax-rejected" : "capture-failed")
@@ -374,6 +356,7 @@ extension RunnerTests {
374356
)
375357
continue
376358
}
359+
guard case let .captured(capture) = attempt.outcome else { continue }
377360

378361
if let sparseReason = Self.sparsePayloadReason(capture.qualityPayload ?? capture.payload) {
379362
if firstFailure == nil { firstFailure = sparseReason }
@@ -429,92 +412,96 @@ extension RunnerTests {
429412
return fallbackPayload
430413
}
431414

432-
/// Marks XCTest-backed snapshot tiers as penalized when one attempt ground past the slow-capture
433-
/// threshold — even a successful one: the next capture of this screen must not re-grind.
434-
private func recordSlowXCTestSnapshotBackendIfNeeded(
435-
_ kind: SnapshotBackendKind,
436-
startedAt: Date,
437-
penaltySuppressed: Bool
438-
) {
439-
guard !penaltySuppressed else { return }
440-
guard kind.usesXCTestAccessibilityChannel else { return }
441-
let elapsed = Date().timeIntervalSince(startedAt)
442-
guard elapsed > snapshotXCTestSlowCaptureThreshold else { return }
443-
penalizeSnapshotXCTestChannel(
444-
bundleId: currentBundleId,
445-
reason: "slow_\(kind.rawValue)_capture_\(Int(elapsed * 1000))ms"
446-
)
447-
}
448-
449-
private func recordXCTestSnapshotBackendFailureIfNeeded(
450-
_ kind: SnapshotBackendKind,
451-
failure: SnapshotCaptureFailure,
452-
penaltySuppressed: Bool
453-
) {
454-
guard !penaltySuppressed else { return }
455-
guard kind.usesXCTestAccessibilityChannel, failure.code == Self.xCTestSnapshotTimeoutCode else { return }
456-
penalizeSnapshotXCTestChannel(
457-
bundleId: currentBundleId,
458-
reason: "\(kind.rawValue)_backend_timeout"
459-
)
460-
}
461-
462415
private func captureWithBackend(
463416
_ kind: SnapshotBackendKind,
464417
app: XCUIApplication,
465418
options: PresentationOptions,
466419
deadline: Date,
467420
treeCaptureSliceBudgetOverride: TimeInterval?
468-
) throws -> SnapshotBackendCapture? {
421+
) throws -> SnapshotBackendAttempt {
469422
let hint = SnapshotPresentation.captureHint(for: options)
423+
var timer = SnapshotPhaseTimer()
470424
let acquisition: SnapshotAcquisition?
471-
switch kind {
472-
case .recursiveTree:
473-
guard
474-
let context = try makeSnapshotTraversalContext(
475-
app: app,
476-
hint: hint,
477-
captureDeadline: deadline,
478-
treeCaptureSliceBudgetOverride: treeCaptureSliceBudgetOverride
479-
)
480-
else {
481-
return nil
482-
}
483-
acquisition = try runMainThreadWork(
484-
timeout: min(treeCaptureSliceBudget, max(0.5, deadline.timeIntervalSinceNow)),
485-
timeoutError: snapshotMainThreadTimeoutError("processing tree snapshot")
486-
) {
487-
hint.isRaw
488-
? try self.rawTreeSnapshotAcquisition(context: context, hint: hint)
489-
: self.recursiveTreeSnapshotAcquisition(context: context, hint: hint)
490-
}
491-
case .querySweep:
492-
acquisition = try runMainThreadWork(
493-
timeout: min(Self.flatInteractiveFallbackBudget, max(0.1, deadline.timeIntervalSinceNow)),
494-
timeoutError: snapshotMainThreadTimeoutError("running query-sweep snapshot")
495-
) {
496-
self.querySweepSnapshotAcquisition(
497-
app: app,
498-
hint: hint,
499-
planDeadline: deadline
500-
)
425+
do {
426+
acquisition = try timer.measure(.acquisition) {
427+
switch kind {
428+
case .recursiveTree:
429+
guard
430+
let context = try self.makeSnapshotTraversalContext(
431+
app: app,
432+
hint: hint,
433+
captureDeadline: deadline,
434+
treeCaptureSliceBudgetOverride: treeCaptureSliceBudgetOverride
435+
)
436+
else {
437+
return nil
438+
}
439+
return try self.runMainThreadWork(
440+
timeout: min(self.treeCaptureSliceBudget, max(0.5, deadline.timeIntervalSinceNow)),
441+
timeoutError: self.snapshotMainThreadTimeoutError("processing tree snapshot")
442+
) {
443+
hint.isRaw
444+
? try self.rawTreeSnapshotAcquisition(context: context, hint: hint)
445+
: self.recursiveTreeSnapshotAcquisition(context: context, hint: hint)
446+
}
447+
case .querySweep:
448+
return try self.runMainThreadWork(
449+
timeout: min(Self.flatInteractiveFallbackBudget, max(0.1, deadline.timeIntervalSinceNow)),
450+
timeoutError: self.snapshotMainThreadTimeoutError("running query-sweep snapshot")
451+
) {
452+
self.querySweepSnapshotAcquisition(
453+
app: app,
454+
hint: hint,
455+
planDeadline: deadline
456+
)
457+
}
458+
case .privateAX:
459+
return self.privateAXSnapshotAcquisition(
460+
app: app,
461+
hint: hint,
462+
deadline: deadline
463+
)
464+
}
501465
}
502-
case .privateAX:
503-
acquisition = privateAXSnapshotAcquisition(
504-
app: app,
505-
hint: hint,
506-
deadline: deadline
466+
} catch let failure as SnapshotCaptureFailure {
467+
return SnapshotBackendAttempt(
468+
outcome: .failed(failure, phase: .acquisition),
469+
timing: timer.timing
507470
)
508471
}
509-
guard let acquisition else { return nil }
510-
guard let capture = SnapshotPresentation.present(acquisition, options: options) else {
511-
throw Self.snapshotProjectionMismatchFailure(
512-
kind,
513-
requested: hint.projection,
514-
acquired: acquisition.hint.projection
472+
473+
guard let acquisition else {
474+
return SnapshotBackendAttempt(
475+
outcome: .noCapture,
476+
timing: timer.timing
515477
)
516478
}
517-
return capture
479+
480+
let presented: SnapshotBackendCapture
481+
do {
482+
presented = try timer.measure(.presentation) {
483+
guard let capture = SnapshotPresentation.present(acquisition, options: options) else {
484+
throw Self.snapshotProjectionMismatchFailure(
485+
kind,
486+
requested: hint.projection,
487+
acquired: acquisition.hint.projection
488+
)
489+
}
490+
return capture
491+
}
492+
} catch let failure as SnapshotCaptureFailure {
493+
return SnapshotBackendAttempt(
494+
outcome: .failed(failure, phase: .presentation),
495+
timing: timer.timing
496+
)
497+
}
498+
499+
var capture = presented
500+
capture.timing = timer.timing
501+
return SnapshotBackendAttempt(
502+
outcome: .captured(capture),
503+
timing: timer.timing
504+
)
518505
}
519506

520507
/// A backend that answers a request with the other projection loses its tier and says why, so
@@ -632,7 +619,8 @@ extension RunnerTests {
632619
reasonCode: reason?.code,
633620
effectiveDepth: capture.effectiveDepth,
634621
collapsedLeafIndexes: Self.collapsedLeafIndexes(payload.nodes ?? []),
635-
customActions: capture.customActions
622+
customActions: capture.customActions,
623+
timing: capture.timing
636624
)
637625
return DataPayload(
638626
// Legacy human text for older daemons that read message instead of snapshotQuality.
@@ -847,6 +835,56 @@ extension RunnerTests {
847835
XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe)?.code, "budget")
848836
}
849837

838+
func testSnapshotQualityCarriesPhaseTimingAtResponseLevel() {
839+
let timing = SnapshotCaptureTiming(acquisitionMs: 12, presentationMs: 34)
840+
let capture = SnapshotBackendCapture(
841+
payload: DataPayload(
842+
nodes: [planTestNode(index: 0, type: "Application", label: "App")],
843+
truncated: false
844+
),
845+
effectiveDepth: nil,
846+
timing: timing
847+
)
848+
849+
let payload = stampedSnapshotPayload(
850+
capture,
851+
backend: .recursiveTree,
852+
state: "healthy",
853+
reason: nil
854+
)
855+
856+
XCTAssertEqual(payload.snapshotQuality?.timing, timing)
857+
XCTAssertEqual(payload.nodes?.count, 1)
858+
}
859+
860+
func testDirectPresentationDoesNotClaimPlanTiming() {
861+
let options = PresentationOptions(
862+
interactiveOnly: false,
863+
depth: nil,
864+
scope: nil,
865+
raw: true
866+
)
867+
let capture = SnapshotPresentation.presentRaw(
868+
SnapshotAcquisition(
869+
hint: SnapshotPresentation.captureHint(for: options),
870+
nodes: [],
871+
truncated: false,
872+
effectiveDepth: nil,
873+
viewport: .infinite
874+
),
875+
options: options
876+
)
877+
878+
let payload = stampedSnapshotPayload(
879+
capture,
880+
backend: .recursiveTree,
881+
state: "healthy",
882+
reason: nil
883+
)
884+
885+
XCTAssertNil(payload.snapshotQuality?.timing)
886+
}
887+
850888
/// The raw plan is derived from what each backend can actually serve, not from a second
851889
/// hand-maintained list. Non-vacuity: flipping `querySweep.supportsRawProjection` to true adds it
852890
/// to the plan and fails the first two assertions — which is exactly the shape of #1797 D4, a

0 commit comments

Comments
 (0)