Replay Nightly #155
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Replay Nightly | |
| on: | |
| schedule: | |
| - cron: '0 3 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| fuzz-iterations: | |
| description: 'Parser fuzz cases per target' | |
| required: false | |
| default: '50000' | |
| fuzz-seed: | |
| description: 'Parser fuzz PRNG seed (defaults to the run number)' | |
| required: false | |
| permissions: | |
| contents: read | |
| actions: read | |
| concurrency: | |
| group: ci-${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Parser fuzz lane (#1414): hostile input into parseArgs, selector parsing, .ad replay | |
| # scripts, batch --steps JSON, and the Maestro compat parser. One invariant — every | |
| # rejection is a typed AppError with a non-empty hint, and no case hangs. Device-free, so | |
| # it rides this nightly rather than owning a workflow; the seed varies per run so the lane | |
| # keeps exploring, and anything it catches is appended to the checked-in corpus that the | |
| # unit lane replays (scripts/fuzz/corpus/regressions.json). | |
| nightly-parser-fuzz: | |
| name: Parser Fuzz Lane | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup toolchain | |
| uses: ./.github/actions/setup-node-pnpm | |
| # A lane whose classifier or watchdog regressed would pass forever. This runs the | |
| # broken-on-purpose targets first and fails unless each violation is caught. | |
| - name: Self-check the harness | |
| uses: ./.github/actions/run-gate | |
| with: | |
| gate: fuzz-parsers | |
| args: | | |
| --self-check | |
| --artifact-dir | |
| .tmp/fuzz/self-check | |
| # Runs even when the self-check fails, so the lane always produces a fuzz envelope too | |
| # (a step that never runs writes no envelope, and monitoring cannot tell that apart from | |
| # a lane that went dark). The job still fails because both steps report their status. | |
| - name: Fuzz parsers | |
| if: always() | |
| env: | |
| FUZZ_ITERATIONS: ${{ github.event.inputs.fuzz-iterations || '50000' }} | |
| FUZZ_SEED: ${{ github.event.inputs.fuzz-seed || github.run_number }} | |
| uses: ./.github/actions/run-gate | |
| with: | |
| gate: fuzz-parsers | |
| args: | | |
| --iterations | |
| ${{ env.FUZZ_ITERATIONS }} | |
| --seed | |
| ${{ env.FUZZ_SEED }} | |
| --artifact-dir | |
| .tmp/fuzz/run | |
| # Uploaded on pass as well as failure: each subdirectory of .tmp/fuzz holds a | |
| # run-envelope.json on #1430's shared lane contract (scripts/lib/lane-envelope.ts) — | |
| # commit/ref/run provenance, tool + config hash, seed, result — which is what freshness | |
| # monitoring reads. | |
| - name: Upload run envelopes and failing cases | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: parser-fuzz-run-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: .tmp/fuzz | |
| if-no-files-found: warn | |
| # Never fails the job on its own: a missing envelope is reported as such rather than | |
| # masking the real failure with a `cat` error. | |
| - name: Summarize | |
| if: always() | |
| run: | | |
| { | |
| echo '### Parser fuzz lane' | |
| found=0 | |
| for envelope in .tmp/fuzz/*/run-envelope.json; do | |
| [ -f "$envelope" ] || continue | |
| found=1 | |
| echo "#### $envelope" | |
| echo '```json' | |
| cat "$envelope" | |
| echo '```' | |
| done | |
| [ "$found" = 1 ] || echo 'No run envelope was produced — the lane failed before it could run.' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| nightly-android: | |
| name: Android Full Emulator Suite | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| env: | |
| AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/android-e2e-state | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup toolchain | |
| uses: ./.github/actions/setup-node-pnpm | |
| - name: Setup Android replay host | |
| id: android-replay-host | |
| uses: ./.github/actions/setup-android-replay-host | |
| with: | |
| package-helpers: 'true' | |
| - name: Restore Android fixture APK | |
| id: fixture-app | |
| uses: ./.github/actions/setup-fixture-app | |
| with: | |
| platform: android | |
| install: 'false' | |
| require-artifact: 'true' | |
| - name: Mark Android emulator setup complete | |
| id: android-setup | |
| run: echo "seconds=$(date +%s)" >> "$GITHUB_OUTPUT" | |
| - name: Run Android full emulator suite | |
| uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0 | |
| with: | |
| api-level: 36 | |
| arch: x86_64 | |
| profile: pixel_7 | |
| target: google_apis_playstore | |
| emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -no-metrics | |
| script: | | |
| # android-emulator-runner executes this with /usr/bin/sh, which is dash on | |
| # ubuntu runners and rejects `-o pipefail`. The serial assignment below is | |
| # validated by the `test -n` guard instead. | |
| set -eu | |
| ANDROID_SERIAL="$(adb devices | awk 'NR > 1 && $2 == "device" { print $1; exit }')" | |
| test -n "$ANDROID_SERIAL" | |
| BOOT_FINISHED_AT="$(date +%s)" | |
| pnpm gate build | |
| pnpm clean:daemon | |
| adb -s "$ANDROID_SERIAL" install -r "${{ steps.fixture-app.outputs.apk-path }}" | |
| node --experimental-strip-types src/bin.ts replay examples/test-app/replays/drag-android.ad --platform android --serial "$ANDROID_SERIAL" --session nightly-android-target-drag --json | |
| pnpm clean:daemon | |
| FULL_STARTED_AT="$(date +%s)" | |
| AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_E2E_TIER=full AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="${{ steps.fixture-app.outputs.apk-path }}" AGENT_DEVICE_FIXTURE_APP_ID="${{ steps.fixture-app.outputs.app-id }}" node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-android-emulator.test.ts | |
| FULL_FINISHED_AT="$(date +%s)" | |
| echo "Android setup before emulator: $((BOOT_FINISHED_AT - ${{ steps.android-setup.outputs.seconds }}))s" | |
| echo "Android full scenario wall time: $((FULL_FINISHED_AT - FULL_STARTED_AT))s" | |
| pnpm gate replay-android --retries 2 --artifacts-dir test/artifacts/replays-android --report-junit test/artifacts/replays-android.junit.xml | |
| - name: Upload Android artifacts | |
| if: always() | |
| uses: ./.github/actions/upload-agent-device-artifacts | |
| with: | |
| artifact-name: replay-nightly-android-artifacts | |
| agent-state-dir: ${{ env.AGENT_DEVICE_STATE_DIR }} | |
| nightly-ios: | |
| name: iOS Replay Suite | |
| runs-on: macos-26 | |
| timeout-minutes: 80 | |
| env: | |
| IOS_RUNTIME_VERSION: '26.2' | |
| AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state | |
| AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived | |
| AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS: '420000' | |
| AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE: agent-device-test-app:///automation?event={event}&payload={payload} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup toolchain | |
| uses: ./.github/actions/setup-node-pnpm | |
| with: | |
| cache-dependency-path: | | |
| pnpm-lock.yaml | |
| examples/test-app/pnpm-lock.yaml | |
| - name: Restore and build iOS XCTest runner | |
| uses: ./.github/actions/setup-apple-runner-build | |
| with: | |
| derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} | |
| cache-key-prefix: ios-runner-prebuilt | |
| cache-key-suffix: -ios-${{ env.IOS_RUNTIME_VERSION }} | |
| gate: swift-runner-ios | |
| xcuitest-platform: ios | |
| xcuitest-destination: generic/platform=iOS Simulator | |
| - name: Boot iOS test simulator | |
| id: ios-simulator | |
| uses: ./.github/actions/boot-ios-test-simulator | |
| with: | |
| runtime-version: ${{ env.IOS_RUNTIME_VERSION }} | |
| preferred-device-name: iPhone 17 Pro | |
| - name: Preflight iOS runner through public CLI | |
| run: | | |
| pnpm clean:daemon | |
| node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json | |
| pnpm clean:daemon | |
| - name: Run iOS simulator replay suite | |
| uses: ./.github/actions/run-gate | |
| with: | |
| gate: replay-ios | |
| args: | | |
| --udid | |
| ${{ steps.ios-simulator.outputs.simulator-udid }} | |
| --retries | |
| 2 | |
| --artifacts-dir | |
| test/artifacts/replays-ios-simulator | |
| --report-junit | |
| test/artifacts/replays-ios-simulator.junit.xml | |
| - name: Fetch current fixture app | |
| id: fixture-app | |
| uses: ./.github/actions/setup-fixture-app | |
| with: | |
| install: 'false' | |
| - name: Report fixture cache source | |
| run: | | |
| echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Prove selector drag reaches its destination on iOS | |
| run: | | |
| xcrun simctl install "${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.fixture-app.outputs.app-path }}" | |
| pnpm clean:daemon | |
| node --experimental-strip-types src/bin.ts replay examples/test-app/replays/drag.ad --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --session nightly-ios-target-drag --json | |
| pnpm clean:daemon | |
| - name: Run full fixture-backed iOS simulator E2E | |
| env: | |
| AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} | |
| AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} | |
| AGENT_DEVICE_IOS_E2E: '1' | |
| AGENT_DEVICE_IOS_E2E_TIER: full | |
| AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} | |
| run: | | |
| pnpm gate build | |
| node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts | |
| - name: Run iOS physical device replay suite | |
| if: env.IOS_UDID != '' | |
| env: | |
| IOS_UDID: ${{ vars.IOS_UDID }} | |
| uses: ./.github/actions/run-gate | |
| with: | |
| gate: replay-ios-device | |
| args: | | |
| --udid | |
| ${{ env.IOS_UDID }} | |
| --retries | |
| 2 | |
| --artifacts-dir | |
| test/artifacts/replays-ios-device | |
| --report-junit | |
| test/artifacts/replays-ios-device.junit.xml | |
| - name: Upload iOS artifacts | |
| if: always() | |
| uses: ./.github/actions/upload-agent-device-artifacts | |
| with: | |
| artifact-name: replay-nightly-ios-artifacts | |
| agent-state-dir: ${{ env.AGENT_DEVICE_STATE_DIR }} | |
| runner-derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} | |
| nightly-macos: | |
| name: macOS Replay Suite | |
| runs-on: macos-26 | |
| timeout-minutes: 80 | |
| env: | |
| AGENT_DEVICE_STATE_DIR: ${{ github.workspace }}/.tmp/agent-device-state | |
| AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/macos-runner-derived | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup toolchain | |
| uses: ./.github/actions/setup-node-pnpm | |
| - name: Restore and build macOS XCTest runner | |
| uses: ./.github/actions/setup-apple-runner-build | |
| with: | |
| derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} | |
| cache-key-prefix: macos-runner-prebuilt | |
| gate: swift-runner-macos | |
| xcuitest-platform: macos | |
| xcuitest-destination: platform=macOS,arch=arm64 | |
| - name: Build macOS helper | |
| uses: ./.github/actions/run-gate | |
| with: { gate: macos-helper } | |
| - name: Run macOS replay suite | |
| uses: ./.github/actions/run-gate | |
| with: | |
| gate: replay-macos | |
| args: | | |
| --retries | |
| 2 | |
| --report-junit | |
| test/artifacts/replays-macos.junit.xml | |
| - name: Upload macOS artifacts | |
| if: always() | |
| uses: ./.github/actions/upload-agent-device-artifacts | |
| with: | |
| artifact-name: replay-nightly-macos-artifacts | |
| agent-state-dir: ${{ env.AGENT_DEVICE_STATE_DIR }} | |
| runner-derived-path: ${{ env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH }} |