|
| 1 | +name: XCTest Nightly |
| 2 | + |
| 3 | +# The full iOS runner XCTest suite (#1781 A7). The PR lane (ios.yml) names a hand-written |
| 4 | +# subset of the target's methods in an `-only-testing:` list; everything outside that list ran |
| 5 | +# nowhere at all. This lane drops the filter and runs the test plan whole, skipping only |
| 6 | +# `testCommand` — which is the runner's server entry point rather than a test (see the step |
| 7 | +# below). |
| 8 | +# |
| 9 | +# No count is quoted here on purpose. `pnpm check:xctest-selection` prints the live split |
| 10 | +# (declared / PR-selected / skipped / nightly-only) and is the only place those numbers are |
| 11 | +# derived rather than remembered. A hand-copied figure in a comment is the same class of |
| 12 | +# defect this lane exists to close: between this file being written and first review, the |
| 13 | +# suite gained three methods and the PR list gained five entries, so every number originally |
| 14 | +# written here was already wrong. |
| 15 | +# |
| 16 | +# Under the lane rule in #1781: |
| 17 | +# |
| 18 | +# - Catches: regressions in the runner-semantics tests the PR list leaves out — selector |
| 19 | +# matching, snapshot traversal, command journal, sequence execution, tv remote, keyboard, |
| 20 | +# scroll/gesture policy. Nothing else executes them. Note the reachable set is smaller |
| 21 | +# than the declared set: `testCommand` is skipped, and the `…OnTvOS` tests sit behind |
| 22 | +# `#if os(tvOS)` and cannot execute on this lane's iOS Simulator destination. |
| 23 | +# - Evidence: the list has never been re-derived since it was written by hand; the tests it |
| 24 | +# omits have been dark for their whole life, so their current state is unmeasured. The |
| 25 | +# first red night is the evidence this lane exists to produce. |
| 26 | +# - Cost: this is a public repository, so nothing here is billed. The real cost is one macOS |
| 27 | +# concurrency slot per night. The Apple runner build shares ios.yml's cache (identical |
| 28 | +# inputs and build variant), but scheduled runs only see DEFAULT-BRANCH caches, so the hit |
| 29 | +# depends on ios.yml's `push: main` run having populated the cache at the same source |
| 30 | +# hash — on a miss this lane also pays a full runner build. Adds nothing to any PR. |
| 31 | +# - Kill criterion: when A7's classification lands — pure Swift decision tests moved to a |
| 32 | +# fast unit target, stale tests deleted, the rest kept on the simulator — this lane goes |
| 33 | +# if the surviving set is small enough to run on every PR. |
| 34 | +# |
| 35 | +# Scheduled + manual only. It is deliberately NOT a PR gate: ios.yml is already over half of |
| 36 | +# this repo's CI spend (#1781 A9), and moving the unrun set onto it would make that worse for |
| 37 | +# tests whose flakiness on a shared runner is unknown. |
| 38 | +# |
| 39 | +# Note on `concurrency` below: it is keyed on `github.ref`, so a manual dispatch on the same |
| 40 | +# ref cancels an in-flight scheduled run. That is the intended trade — an operator asking for |
| 41 | +# a run wants the fresh one — but it means "dispatch while the nightly is running" loses the |
| 42 | +# nightly's result rather than queueing behind it. |
| 43 | + |
| 44 | +on: |
| 45 | + schedule: |
| 46 | + - cron: '30 4 * * *' |
| 47 | + workflow_dispatch: |
| 48 | + |
| 49 | +permissions: |
| 50 | + contents: read |
| 51 | + actions: read |
| 52 | + |
| 53 | +concurrency: |
| 54 | + group: ci-${{ github.workflow }}-${{ github.ref }} |
| 55 | + cancel-in-progress: true |
| 56 | + |
| 57 | +jobs: |
| 58 | + full-suite: |
| 59 | + name: iOS Runner Full XCTest Suite |
| 60 | + runs-on: macos-26 |
| 61 | + # Measured, not guessed. A full local run of this exact command on matching toolchain |
| 62 | + # (Xcode 26.2, iOS 26.2 Simulator) executed 154 tests in 120s wall — the suite itself is |
| 63 | + # ~2 minutes, not the bulk of the job. What dominates the budget is the runner build on a |
| 64 | + # derived-data cache MISS, plus checkout and simulator boot. ios.yml carries 80 minutes |
| 65 | + # while doing strictly more than this lane (same build and boot, plus replays, prepare, |
| 66 | + # and the fixture E2E), so half of that is generous headroom for a cache-miss night and |
| 67 | + # still caps a wedged run at 45 minutes instead of two hours of held macOS slot. |
| 68 | + timeout-minutes: 45 |
| 69 | + env: |
| 70 | + # Must match ios.yml. The runtime version rides in the derived-data cache key, and |
| 71 | + # AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS is BOTH an input to the cache's build-variant |
| 72 | + # hash and the `-D AGENT_DEVICE_RUNNER_UNIT_TESTS` compile flag that makes the RunnerTests |
| 73 | + # methods exist at all. Dropping it would build a different variant (cache miss) whose |
| 74 | + # bundle contains no tests, and `xcodebuild` reports running zero tests as success — the |
| 75 | + # "silent 0-test green" this lane would then be. The test-count assertion below is the |
| 76 | + # backstop for exactly that. |
| 77 | + IOS_RUNTIME_VERSION: '26.2' |
| 78 | + AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS: '1' |
| 79 | + AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived |
| 80 | + RESULT_BUNDLE_PATH: ${{ github.workspace }}/.tmp/xctest-nightly/RunnerTests.xcresult |
| 81 | + RESULT_SUMMARY_PATH: ${{ github.workspace }}/.tmp/xctest-nightly/test-results-summary.json |
| 82 | + steps: |
| 83 | + - name: Checkout |
| 84 | + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 |
| 85 | + |
| 86 | + - name: Setup toolchain |
| 87 | + uses: ./.github/actions/setup-node-pnpm |
| 88 | + with: |
| 89 | + cache-dependency-path: | |
| 90 | + pnpm-lock.yaml |
| 91 | + examples/test-app/pnpm-lock.yaml |
| 92 | +
|
| 93 | + # Byte-identical inputs to ios.yml's step, so both lanes restore the same cache entry |
| 94 | + # and only the first one to run on a commit pays for the build. |
| 95 | + - name: Restore and build iOS XCTest runner |
| 96 | + uses: ./.github/actions/setup-apple-runner-build |
| 97 | + with: |
| 98 | + derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} |
| 99 | + cache-key-prefix: ios-runner-prebuilt |
| 100 | + cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }} |
| 101 | + gate: swift-runner-ios |
| 102 | + xcuitest-platform: ios |
| 103 | + xcuitest-destination: generic/platform=iOS Simulator |
| 104 | + |
| 105 | + - name: Boot iOS test simulator |
| 106 | + id: ios-simulator |
| 107 | + uses: ./.github/actions/boot-ios-test-simulator |
| 108 | + with: |
| 109 | + runtime-version: ${{ env.IOS_RUNTIME_VERSION }} |
| 110 | + preferred-device-name: iPhone 17 Pro |
| 111 | + |
| 112 | + # Same command as ios.yml's targeted step minus every `-only-testing:` flag, so the |
| 113 | + # xctestrun's own test plan decides what runs — with one exception. |
| 114 | + # |
| 115 | + # `RunnerTests/testCommand` is not a test. It is the runner's server entry point: it |
| 116 | + # opens an NWListener and blocks in `XCTWaiter.wait(timeout: 24 * 60 * 60)` until a |
| 117 | + # client closes the session, which is exactly how production drives it |
| 118 | + # (src/platforms/apple/core/runner/runner-session.ts always passes it as the sole |
| 119 | + # `-only-testing:`). It compiles unconditionally — the `#if AGENT_DEVICE_RUNNER_UNIT_TESTS` |
| 120 | + # block ends well above it — so an unfiltered run reaches it in alphabetical order and |
| 121 | + # hangs the job until `timeout-minutes`. Its one escape hatch, |
| 122 | + # AGENT_DEVICE_RUNNER_NOOP_STARTUP, is an environment variable, and the comment above |
| 123 | + # the flag records that env plumbing into a simulator test process is not dependable; |
| 124 | + # `-skip-testing:` is the lever that works from here. |
| 125 | + # |
| 126 | + # A typo in that identifier silently re-arms the hang, so |
| 127 | + # `pnpm check:xctest-selection` validates `-skip-testing:` exactly like `-only-testing:`. |
| 128 | + - name: Run the full iOS runner XCTest suite |
| 129 | + run: | |
| 130 | + set -euo pipefail |
| 131 | + XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" |
| 132 | + test -n "$XCTESTRUN_PATH" |
| 133 | + mkdir -p "$(dirname "$RESULT_BUNDLE_PATH")" |
| 134 | + xcodebuild test-without-building \ |
| 135 | + -xctestrun "$XCTESTRUN_PATH" \ |
| 136 | + -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ |
| 137 | + -skip-testing:AgentDeviceRunnerUITests/RunnerTests/testCommand \ |
| 138 | + -resultBundlePath "$RESULT_BUNDLE_PATH" |
| 139 | +
|
| 140 | + # Best-effort and never the job's verdict on its own; the step below is what asserts. |
| 141 | + # `--compact` first because a red night's summary is the large one, and the job summary |
| 142 | + # has a 1 MiB cap — the plain form is the fallback if an Xcode version rejects the flag. |
| 143 | + - name: Extract the test-results summary |
| 144 | + if: always() |
| 145 | + run: | |
| 146 | + set -uo pipefail |
| 147 | + [ -d "$RESULT_BUNDLE_PATH" ] || exit 0 |
| 148 | + xcrun xcresulttool get test-results summary --path "$RESULT_BUNDLE_PATH" --compact \ |
| 149 | + > "$RESULT_SUMMARY_PATH" 2>/dev/null || |
| 150 | + xcrun xcresulttool get test-results summary --path "$RESULT_BUNDLE_PATH" \ |
| 151 | + > "$RESULT_SUMMARY_PATH" 2>/dev/null || |
| 152 | + rm -f "$RESULT_SUMMARY_PATH" |
| 153 | + exit 0 |
| 154 | +
|
| 155 | + # Reports the run AND is the lane's own liveness check. `xcodebuild` exits 0 when a |
| 156 | + # selection matches nothing, so "green" and "ran no tests" are the same log tail: a |
| 157 | + # build variant without the unit-test compile flag, an empty test plan, or a renamed |
| 158 | + # target would all read as a healthy night. Asserting the run executed tests is what |
| 159 | + # tells those apart. The failure list is capped so a badly red night cannot blow the |
| 160 | + # 1 MiB job-summary limit and lose the headline with it. |
| 161 | + - name: Report the run and assert it executed tests |
| 162 | + if: always() |
| 163 | + run: | |
| 164 | + set -euo pipefail |
| 165 | + if [ ! -s "$RESULT_SUMMARY_PATH" ]; then |
| 166 | + echo 'No usable test-results summary was produced, so nothing here can show a test ran.' |
| 167 | + echo 'The suite step above failed before or during the run; read its log first.' |
| 168 | + echo '### iOS runner full XCTest suite' >> "$GITHUB_STEP_SUMMARY" |
| 169 | + echo 'No result-bundle summary; see the run log and the uploaded bundle.' >> "$GITHUB_STEP_SUMMARY" |
| 170 | + exit 1 |
| 171 | + fi |
| 172 | + node --experimental-strip-types scripts/xctest-run-summary.ts |
| 173 | +
|
| 174 | + - name: Upload result bundle |
| 175 | + if: always() |
| 176 | + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 |
| 177 | + with: |
| 178 | + name: xctest-nightly-results-${{ github.run_id }}-${{ github.run_attempt }} |
| 179 | + path: .tmp/xctest-nightly |
| 180 | + if-no-files-found: warn |
0 commit comments