diff --git a/.github/PERFORMANCE.md b/.github/PERFORMANCE.md index c279e5e62..55a2613d0 100644 --- a/.github/PERFORMANCE.md +++ b/.github/PERFORMANCE.md @@ -33,11 +33,33 @@ Changed benchmark definitions run two head-only measurements as a new baseline, without executing the old base app or publishing an invented paired baseline. This also handles the first rollout of a new runner protocol. +## Saved apps and measurement reruns + +The workflow separates preparation, platform builds, platform measurements and +collection. Each measurement job downloads the immutable app artifact ID produced +by its build job; base and head still run together on one machine. A changed suite +builds only head. An identical base/head SHA reuses the same binary for both sides. +Otherwise each revision is built from its own checkout with the same build script. + +Use GitHub's **Re-run job and dependent jobs** on `measure-android` or +`measure-ios` to repeat measurements without rebuilding successful ancestors. +Collection downloads the exact result IDs from those jobs. An untouched platform +keeps its original attempt; the report records and links both platform measurement +attempts and their app build attempts. Rerunning all jobs intentionally rebuilds. +Apps and results expire after 30 days; an expired artifact requires a new build. + +The app artifact includes base/head SHAs, suite hashes, Release configuration, +architecture and toolchain metadata. iOS apps are tar archives to preserve +permissions and symlinks. Gradle's basic cache is the sole Android cache owner; +Gradle still checks source/task inputs, while exact app reuse is by artifact ID. +There is no new iOS compiler cache. First-run speed or CI stability improvements +have not been measured; app reuse specifically avoids build work on manual reruns. + ## Artifacts and publishing The canonical artifact is `performance-report-`: raw JSON for every measured process and discarded calibration plan, plus `performance-report.json` with repository, revisions, -workflow run and attempt provenance. Artifacts remain available for 30 days. +workflow run and attempt provenance, original build metadata and measurement artifact IDs. Artifacts remain available for 30 days. The PR comment links its exact immutable artifact ID; downloads require GitHub access. An agent can inspect the JSON instead of scraping the rendered table. @@ -46,17 +68,17 @@ forks and main runs. It downloads the exact artifact from the triggering attempt validates bounded JSON against GitHub's run and current PR metadata, and computes the comparison, Markdown and Bencher values from raw samples. It never installs or executes PR code or app artifacts. Docs-only and cancelled runs skip -publication. Relevant failures remain failures. Stale PR results are skipped. +publication. Markdown/MDX-only edits also skip measurements inside package/app directories. Relevant failures remain failures. Stale PR results are skipped. The trusted publisher uses `BENCHER_KEY` as an Actions secret. Its CLI version and binary digest are pinned. Bencher receives median latency values without invented bounds; its JSON adapter requires only `value`. PR publications seed both measured platform baselines at `baseline-` before recording the head at -`pr-`. Main runs record main history. User comments are never edited. +`pr-`. Main runs record main history. Bencher receives history only: it does not post a second GitHub comment or create alert-driven checks. The trusted renderer owns the one PR comment. User comments are never edited. The raw-manifest publisher and producer land together in the first cleanup PR. Until that PR reaches the default branch, the previous trusted reporter cannot -consume its new manifest. Merge that PR before running later stack revisions; +consume its new manifest. Merge the complete producer/publisher stack before relying on its new provenance contract; there is no permanent old-schema reporting path. Existing raw app results may contain extra summary fields, which the raw parser ignores. @@ -71,3 +93,17 @@ bound boot time to five minutes and individual install/launch commands to two. The benchmark app shares real Nitro test packages without Harness/navigation UI. Correctness remains in `apps/example`. See the [benchmark app README](../apps/benchmark/README.md) for local build and run commands. + +## Remaining upstream warnings + +The benchmark now uses the supported ReactModuleInfo constructor and matching +`reactContext` parameter name. Its Gradle property assignments use current syntax. +AGP/RN and Nitro package Gradle warnings remain outside that focused app edit. +Android uses supported `-gpu swiftshader` with required KVM CPU acceleration. + +The emulator action hardcodes `cmdline-tools/latest`. Older hosted SDK managers +can still emit the XML v3/v4 metadata warning. Updating only the action does not +replace an installed SDK manager; latest command-line tools 23 also deprecates +`sdkmanager` in favor of the Android CLI. This change logs the selected SDK manager +and version, and leaves provisioning to the runner/action rather than patching its +SDK layout. This inherited warning is not suppressed. diff --git a/.github/workflows/performance-report.yml b/.github/workflows/performance-report.yml index 0632084a1..c4374a685 100644 --- a/.github/workflows/performance-report.yml +++ b/.github/workflows/performance-report.yml @@ -7,7 +7,6 @@ on: permissions: actions: read - checks: write contents: read pull-requests: write @@ -30,7 +29,7 @@ jobs: - name: Download performance report id: download if: steps.select.outputs.artifact_id != '' - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: artifact-ids: ${{ steps.select.outputs.artifact_id }} path: untrusted-artifact @@ -60,6 +59,7 @@ jobs: --expected-repository "$GITHUB_REPOSITORY" \ --trusted-workflow-event "$GITHUB_EVENT_PATH" \ --artifact-id "${{ steps.select.outputs.artifact_id }}" \ + --trusted-artifacts trusted-artifacts.json \ "${TRUSTED_PR_ARGUMENTS[@]}" - name: Post paired comparison to the PR @@ -79,10 +79,9 @@ jobs: run: | printf '%s %s\n' 'c2d3a6a7fae654246134e5ced1408bdb9ba4e198b0ac3b903af17a06574a7e08' "$(command -v bencher)" | sha256sum --check - - - name: Publish to Bencher and GitHub + - name: Publish Bencher history if: steps.download.outcome == 'success' && steps.validate.outputs.stale != 'true' env: BENCHER_API_KEY: ${{ secrets.BENCHER_KEY }} BENCHER_PROJECT: nitro - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: bun scripts/performance/publish.ts --directory validated-report diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 39af7153d..ef62a1ece 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -64,7 +64,9 @@ jobs: 'apps/benchmark' \ 'packages/react-native-nitro-modules' \ 'packages/react-native-nitro-test' \ - 'packages/react-native-nitro-test-external'; then + 'packages/react-native-nitro-test-external' \ + ':(glob,exclude)**/*.md' \ + ':(glob,exclude)**/*.mdx'; then RELEVANT=false fi @@ -91,25 +93,24 @@ jobs: bun benchmark typecheck bun benchmark lint-ci - android: + build-android: needs: prepare if: needs.prepare.outputs.relevant == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 90 - env: - BASE_SHA: ${{ needs.prepare.outputs.base_sha }} - HEAD_SHA: ${{ needs.prepare.outputs.head_sha }} + timeout-minutes: 40 + outputs: + artifact_id: ${{ steps.upload.outputs.artifact-id }} steps: - name: Checkout base uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: - ref: ${{ env.BASE_SHA }} + ref: ${{ needs.prepare.outputs.base_sha }} path: base - name: Checkout head uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - ref: ${{ env.HEAD_SHA }} + ref: ${{ needs.prepare.outputs.head_sha }} path: head - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: @@ -118,7 +119,63 @@ jobs: with: distribution: zulu java-version: 17 - java-package: jdk + - uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.0.0 + with: + cache-provider: basic + + - name: Build and package exact APKs + run: | + mkdir -p apps + bun head/scripts/performance/build-metadata.ts \ + --platform android --base-root base --head-root head \ + --architecture x86_64 --toolchain "$(java -version 2>&1 | head -1) / NDK 29.0.14206865" \ + --output apps/build.json + bash head/scripts/performance/build-android.sh "$GITHUB_WORKSPACE/head" + cp head/apps/benchmark/android/app/build/outputs/apk/release/app-release.apk apps/head.apk + if [[ "$(jq -r .baseSha apps/build.json)" == "$(jq -r .headSha apps/build.json)" ]]; then + cp apps/head.apk apps/base.apk + elif [[ "$(jq -r .baseSuiteHash apps/build.json)" == "$(jq -r .headSuiteHash apps/build.json)" ]]; then + bash head/scripts/performance/build-android.sh "$GITHUB_WORKSPACE/base" + cp base/apps/benchmark/android/app/build/outputs/apk/release/app-release.apk apps/base.apk + fi + - name: Save immutable Android apps + id: upload + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-apps-android-${{ github.run_attempt }} + path: apps + if-no-files-found: error + retention-days: 30 + compression-level: 0 + + measure-android: + needs: [prepare, build-android] + runs-on: ubuntu-24.04 + timeout-minutes: 75 + outputs: + artifact_id: ${{ steps.upload.outputs.artifact-id }} + env: + BASE_SHA: ${{ needs.prepare.outputs.base_sha }} + HEAD_SHA: ${{ needs.prepare.outputs.head_sha }} + BUILD_ARTIFACT_ID: ${{ needs.build-android.outputs.artifact_id }} + steps: + - name: Checkout measurement controller + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ env.HEAD_SHA }} + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 + with: + distribution: zulu + java-version: 17 + - name: Download exact Android apps + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + artifact-ids: ${{ env.BUILD_ARTIFACT_ID }} + path: apps - name: Enable KVM for benchmark measurements run: | @@ -131,85 +188,61 @@ jobs: stat --format='%A %U:%G %n' /dev/kvm test -r /dev/kvm && test -w /dev/kvm - - name: Select measurement controller - run: echo "PERF_CONTROLLER_ROOT=$GITHUB_WORKSPACE/head" >> "$GITHUB_ENV" - - - name: Build head benchmark APK - working-directory: head - run: | - bun install --frozen-lockfile - cd apps/benchmark/android - ./gradlew :app:assembleRelease --no-daemon --no-build-cache -PreactNativeArchitectures=x86_64 - - - name: Build base benchmark APK - working-directory: base + - name: Report Android command-line tools run: | - bun install --frozen-lockfile - cd apps/benchmark/android - ./gradlew :app:assembleRelease --no-daemon --no-build-cache -PreactNativeArchitectures=x86_64 - - - name: Select base artifact - run: | - echo "BASE_APP=$GITHUB_WORKSPACE/base/apps/benchmark/android/app/build/outputs/apk/release/app-release.apk" >> "$GITHUB_ENV" - echo "BASE_ROOT=$GITHUB_WORKSPACE/base" >> "$GITHUB_ENV" + echo "Emulator action SDK manager: $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" + cat "$ANDROID_HOME/cmdline-tools/latest/source.properties" + "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --version - name: Run paired Android benchmarks - uses: reactivecircus/android-emulator-runner@4c44018e59b437e86cdfc41da381398f93ed8808 # v2 + uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2 with: api-level: 36 arch: x86_64 profile: pixel_7 disable-animations: true - # Never silently fall back to software CPU emulation: it is slow and - # its measurements are not comparable to the KVM-backed testbed. + # CPU acceleration is required, independently of software GPU rendering. disable-linux-hw-accel: false emulator-boot-timeout: 300 - pre-emulator-launch-script: '"$ANDROID_HOME/emulator/emulator" -accel-check' - emulator-options: -accel on -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -camera-back none + pre-emulator-launch-script: '"$ANDROID_HOME/emulator/emulator" -version; "$ANDROID_HOME/emulator/emulator" -accel-check' + emulator-options: -accel on -no-window -gpu swiftshader -no-snapshot -noaudio -no-boot-anim -camera-back none script: >- - bun "$PERF_CONTROLLER_ROOT/scripts/performance/run-sequence.ts" - --platform android - --base-app "$BASE_APP" - --head-app "$GITHUB_WORKSPACE/head/apps/benchmark/android/app/build/outputs/apk/release/app-release.apk" - --base-root "$BASE_ROOT" - --head-root "$GITHUB_WORKSPACE/head" - --base-sha "$BASE_SHA" - --head-sha "$HEAD_SHA" - --output-directory "$GITHUB_WORKSPACE/performance-android" + bun scripts/performance/run-sequence.ts + --platform android --base-app apps/base.apk --head-app apps/head.apk + --build-metadata apps/build.json + --base-sha "$BASE_SHA" --head-sha "$HEAD_SHA" + --output-directory performance-android --device-id "$(adb get-serialno)" - --device "Pixel 7 emulator" - --os-version "Android 16 / API 36" - --architecture x86_64 - --toolchain "JDK 17 / NDK 29.0.14206865 / KVM" - + --device "Pixel 7 emulator / KVM" --os-version "Android 16 / API 36" + --architecture x86_64 --toolchain "$(jq -r .toolchain apps/build.json)" - name: Upload Android results + id: upload if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: performance-android + name: performance-android-${{ github.run_attempt }} path: performance-android if-no-files-found: warn retention-days: 30 - ios: + build-ios: needs: prepare if: needs.prepare.outputs.relevant == 'true' runs-on: macos-26 - timeout-minutes: 120 - env: - BASE_SHA: ${{ needs.prepare.outputs.base_sha }} - HEAD_SHA: ${{ needs.prepare.outputs.head_sha }} + timeout-minutes: 60 + outputs: + artifact_id: ${{ steps.upload.outputs.artifact-id }} steps: - name: Checkout base uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: - ref: ${{ env.BASE_SHA }} + ref: ${{ needs.prepare.outputs.base_sha }} path: base - name: Checkout head uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - ref: ${{ env.HEAD_SHA }} + ref: ${{ needs.prepare.outputs.head_sha }} path: head - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: @@ -218,65 +251,66 @@ jobs: with: ruby-version: 3.3.0 working-directory: head/apps/benchmark - - name: Select Xcode 26.5 run: sudo xcode-select -s /Applications/Xcode_26.5.app/Contents/Developer - - - name: Select measurement controller - run: echo "PERF_CONTROLLER_ROOT=$GITHUB_WORKSPACE/head" >> "$GITHUB_ENV" - - - name: Build head benchmark app - working-directory: head - run: | - bun install --frozen-lockfile - cd apps/benchmark - bundle install - bun pods - cd ios - xcodebuild \ - CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ \ - -derivedDataPath build-benchmark \ - -workspace NitroBenchmark.xcworkspace \ - -scheme NitroBenchmark \ - -configuration Release \ - -sdk iphonesimulator \ - -destination 'generic/platform=iOS Simulator' \ - ARCHS=arm64 \ - ONLY_ACTIVE_ARCH=YES \ - CODE_SIGNING_ALLOWED=NO \ - COMPILER_INDEX_STORE_ENABLE=NO \ - build - - - name: Build base benchmark app - working-directory: base + - name: Build and package exact simulator apps run: | - bun install --frozen-lockfile - cd apps/benchmark - bundle install - bun pods - cd ios - xcodebuild \ - CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ \ - -derivedDataPath build-benchmark \ - -workspace NitroBenchmark.xcworkspace \ - -scheme NitroBenchmark \ - -configuration Release \ - -sdk iphonesimulator \ - -destination 'generic/platform=iOS Simulator' \ - ARCHS=arm64 \ - ONLY_ACTIVE_ARCH=YES \ - CODE_SIGNING_ALLOWED=NO \ - COMPILER_INDEX_STORE_ENABLE=NO \ - build - - - name: Select base artifact - run: | - echo "BASE_APP=$GITHUB_WORKSPACE/base/apps/benchmark/ios/build-benchmark/Build/Products/Release-iphonesimulator/NitroBenchmark.app" >> "$GITHUB_ENV" - echo "BASE_ROOT=$GITHUB_WORKSPACE/base" >> "$GITHUB_ENV" + mkdir -p apps + bun head/scripts/performance/build-metadata.ts \ + --platform ios --base-root base --head-root head \ + --architecture arm64 --toolchain "$(xcodebuild -version | tr '\n' ' ')" \ + --output apps/build.json + bash head/scripts/performance/build-ios.sh "$GITHUB_WORKSPACE/head" + # Tar preserves the executable bits and symlinks inside each .app. + tar -czf apps/head.app.tar.gz -C head/apps/benchmark/ios/build-benchmark/Build/Products/Release-iphonesimulator NitroBenchmark.app + if [[ "$(jq -r .baseSha apps/build.json)" == "$(jq -r .headSha apps/build.json)" ]]; then + cp apps/head.app.tar.gz apps/base.app.tar.gz + elif [[ "$(jq -r .baseSuiteHash apps/build.json)" == "$(jq -r .headSuiteHash apps/build.json)" ]]; then + bash head/scripts/performance/build-ios.sh "$GITHUB_WORKSPACE/base" + tar -czf apps/base.app.tar.gz -C base/apps/benchmark/ios/build-benchmark/Build/Products/Release-iphonesimulator NitroBenchmark.app + fi + - name: Save immutable iOS apps + id: upload + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-apps-ios-${{ github.run_attempt }} + path: apps + if-no-files-found: error + retention-days: 30 + compression-level: 0 + measure-ios: + needs: [prepare, build-ios] + runs-on: macos-26 + timeout-minutes: 90 + outputs: + artifact_id: ${{ steps.upload.outputs.artifact-id }} + env: + BASE_SHA: ${{ needs.prepare.outputs.base_sha }} + HEAD_SHA: ${{ needs.prepare.outputs.head_sha }} + BUILD_ARTIFACT_ID: ${{ needs.build-ios.outputs.artifact_id }} + steps: + - name: Checkout measurement controller + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ env.HEAD_SHA }} + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Select Xcode 26.5 + run: sudo xcode-select -s /Applications/Xcode_26.5.app/Contents/Developer + - name: Download exact iOS apps + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + artifact-ids: ${{ env.BUILD_ARTIFACT_ID }} + path: apps - name: Run paired iOS benchmarks run: | set -euo pipefail + mkdir -p apps/base apps/head + if [[ -f apps/base.app.tar.gz ]]; then tar -xzf apps/base.app.tar.gz -C apps/base; fi + tar -xzf apps/head.app.tar.gz -C apps/head DEVICE_ID=$(xcrun simctl create \ 'Nitro Performance' \ 'com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro' \ @@ -289,105 +323,82 @@ jobs: }); if (await child.exited !== 0) throw new Error("iOS simulator did not boot successfully within five minutes."); ' - bun "$PERF_CONTROLLER_ROOT/scripts/performance/run-sequence.ts" \ - --platform ios \ - --base-app "$BASE_APP" \ - --head-app "$GITHUB_WORKSPACE/head/apps/benchmark/ios/build-benchmark/Build/Products/Release-iphonesimulator/NitroBenchmark.app" \ - --base-root "$BASE_ROOT" \ - --head-root "$GITHUB_WORKSPACE/head" \ - --base-sha "$BASE_SHA" \ - --head-sha "$HEAD_SHA" \ - --output-directory "$GITHUB_WORKSPACE/performance-ios" \ - --device-id "$DEVICE_ID" \ - --device 'iPhone 17 Pro simulator' \ - --os-version 'iOS 26.5' \ - --architecture arm64 \ - --toolchain 'Xcode 26.5' - + bun scripts/performance/run-sequence.ts \ + --platform ios --base-app apps/base/NitroBenchmark.app --head-app apps/head/NitroBenchmark.app \ + --build-metadata apps/build.json \ + --base-sha "$BASE_SHA" --head-sha "$HEAD_SHA" \ + --output-directory performance-ios --device-id "$DEVICE_ID" \ + --device 'iPhone 17 Pro simulator' --os-version 'iOS 26.5' \ + --architecture arm64 --toolchain "$(jq -r .toolchain apps/build.json)" - name: Delete simulator if: always() && env.DEVICE_ID != '' run: | xcrun simctl shutdown "$DEVICE_ID" || true xcrun simctl delete "$DEVICE_ID" || true - - name: Upload iOS results + id: upload if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: performance-ios + name: performance-ios-${{ github.run_attempt }} path: performance-ios if-no-files-found: warn retention-days: 30 nitro-performance: name: nitro-performance - needs: [prepare, android, ios] + needs: [prepare, measure-android, measure-ios] if: always() && !cancelled() && needs.prepare.outputs.relevant == 'true' runs-on: ubuntu-24.04 steps: - - name: Require successful preparation - if: needs.prepare.result != 'success' - run: exit 1 - - name: Require successful platform runs - if: needs.prepare.outputs.relevant == 'true' env: - ANDROID_RESULT: ${{ needs.android.result }} - IOS_RESULT: ${{ needs.ios.result }} + PREPARE_RESULT: ${{ needs.prepare.result }} + ANDROID_RESULT: ${{ needs.measure-android.result }} + IOS_RESULT: ${{ needs.measure-ios.result }} run: | - if [[ "$ANDROID_RESULT" != "success" || "$IOS_RESULT" != "success" ]]; then - echo "Android: $ANDROID_RESULT, iOS: $IOS_RESULT" >&2 + if [[ "$PREPARE_RESULT" != success || "$ANDROID_RESULT" != success || "$IOS_RESULT" != success ]]; then + echo "Prepare: $PREPARE_RESULT, Android: $ANDROID_RESULT, iOS: $IOS_RESULT" >&2 exit 1 fi - - name: Checkout report tooling - if: needs.prepare.outputs.relevant == 'true' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} ref: ${{ needs.prepare.outputs.head_sha }} - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - if: needs.prepare.outputs.relevant == 'true' with: bun-version: 1.3.14 + # Job outputs deliberately retain an untouched platform's earlier artifact + # when only the other measurement job and its dependents are rerun. - name: Download Android results - if: needs.prepare.outputs.relevant == 'true' - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: performance-android - path: artifacts/android + artifact-ids: ${{ needs.measure-android.outputs.artifact_id }} + path: performance-report/raw/android - name: Download iOS results - if: needs.prepare.outputs.relevant == 'true' - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: performance-ios - path: artifacts/ios - + artifact-ids: ${{ needs.measure-ios.outputs.artifact_id }} + path: performance-report/raw/ios - name: Build aggregate report - if: needs.prepare.outputs.relevant == 'true' env: REPOSITORY: ${{ github.repository }} PR_NUMBER: ${{ needs.prepare.outputs.pr_number }} + ANDROID_ARTIFACT_ID: ${{ needs.measure-android.outputs.artifact_id }} + IOS_ARTIFACT_ID: ${{ needs.measure-ios.outputs.artifact_id }} run: | - mkdir -p performance-report/raw/android performance-report/raw/ios - cmp artifacts/android/suite.json artifacts/ios/suite.json + cmp performance-report/raw/android/suite.json performance-report/raw/ios/suite.json bun scripts/performance/report.ts \ - --suite artifacts/android/suite.json \ + --suite performance-report/raw/android/suite.json \ + --raw-directory performance-report/raw \ --output performance-report/performance-report.json \ - --repository "$REPOSITORY" \ - --event-name "${{ github.event_name }}" \ + --repository "$REPOSITORY" --event-name "${{ github.event_name }}" \ --pull-request "$PR_NUMBER" \ --base-sha "${{ needs.prepare.outputs.base_sha }}" \ --head-sha "${{ needs.prepare.outputs.head_sha }}" - for PLATFORM in android ios; do - for RESULT in artifacts/"$PLATFORM"/{base,head,calibration}-*.json; do - if [[ -f "$RESULT" ]]; then cp "$RESULT" performance-report/raw/"$PLATFORM"/; fi - done - done - - name: Upload aggregate report - if: needs.prepare.outputs.relevant == 'true' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: performance-report-${{ github.run_attempt }} path: performance-report diff --git a/apps/benchmark/README.md b/apps/benchmark/README.md index eef4f16db..0716285ee 100644 --- a/apps/benchmark/README.md +++ b/apps/benchmark/README.md @@ -14,12 +14,8 @@ Debug benchmark or publish anything to Bencher. From the repository root: ```sh -bun install --frozen-lockfile -bun benchmark build:android - -bun benchmark bundle-install -bun benchmark pods -bun benchmark build:ios +bash scripts/performance/build-android.sh "$PWD" +bash scripts/performance/build-ios.sh "$PWD" ``` Both platforms use the normal `Release` configuration, an embedded optimized @@ -86,8 +82,10 @@ the timed batch. Operation-induced allocations remain inside it. See [performance CI](../../.github/PERFORMANCE.md) for observed comparisons, process variability, raw artifacts, trusted reporting, and Bencher publishing. -Base and head are built independently. Same-revision scheduled/manual runs show -baseline variation. Performance remains report-only. +CI retains exact app artifacts for measurement-only reruns. It builds both +revisions when comparable, reuses one binary for identical SHAs, and builds only +head when definitions changed. Same-revision scheduled/manual runs show baseline +variation. Performance remains report-only. The example's former benchmark screen and TurboModule control have moved here. No public Nitro API changes are needed. App dependency versions initially match diff --git a/apps/benchmark/android/app/build.gradle b/apps/benchmark/android/app/build.gradle index 9b7c66243..9ab7a195c 100644 --- a/apps/benchmark/android/app/build.gradle +++ b/apps/benchmark/android/app/build.gradle @@ -77,11 +77,11 @@ def enableProguardInReleaseBuilds = true def jscFlavor = 'io.github.react-native-community:jsc-android:2026004.+' android { - ndkVersion rootProject.ext.ndkVersion + ndkVersion = rootProject.ext.ndkVersion buildToolsVersion rootProject.ext.buildToolsVersion compileSdk rootProject.ext.compileSdkVersion - namespace "com.margelo.nitrobenchmark" + namespace = "com.margelo.nitrobenchmark" defaultConfig { applicationId "com.margelo.nitrobenchmark" minSdkVersion rootProject.ext.minSdkVersion @@ -99,13 +99,13 @@ android { } buildTypes { debug { - signingConfig signingConfigs.debug + signingConfig = signingConfigs.debug } release { debuggable false // Caution! In production, you need to generate your own keystore file. // see https://reactnative.dev/docs/signed-apk-android. - signingConfig signingConfigs.debug + signingConfig = signingConfigs.debug minifyEnabled enableProguardInReleaseBuilds proguardFiles getDefaultProguardFile("proguard-android.txt"), "proguard-rules.pro" } diff --git a/apps/benchmark/android/app/src/main/java/com/nitroexample/exampleturbomodule/ExampleTurboModulePackage.kt b/apps/benchmark/android/app/src/main/java/com/nitroexample/exampleturbomodule/ExampleTurboModulePackage.kt index 0ac5c5e9c..68af61a2e 100644 --- a/apps/benchmark/android/app/src/main/java/com/nitroexample/exampleturbomodule/ExampleTurboModulePackage.kt +++ b/apps/benchmark/android/app/src/main/java/com/nitroexample/exampleturbomodule/ExampleTurboModulePackage.kt @@ -7,9 +7,9 @@ import com.facebook.react.module.model.ReactModuleInfo import com.facebook.react.module.model.ReactModuleInfoProvider class ExampleTurboModulePackage: BaseReactPackage() { - override fun getModule(name: String, context: ReactApplicationContext): NativeModule? { + override fun getModule(name: String, reactContext: ReactApplicationContext): NativeModule? { if (name == ExampleTurboModuleModule.NAME) { - return ExampleTurboModuleModule(context) + return ExampleTurboModuleModule(reactContext) } else { return null } @@ -21,7 +21,6 @@ class ExampleTurboModulePackage: BaseReactPackage() { ExampleTurboModuleModule.NAME, false, // canOverrideExistingModule false, // needsEagerInit - true, // hasConstants false, // isCxxModule true // isTurboModule ) diff --git a/scripts/performance/app-isolation.test.ts b/scripts/performance/app-isolation.test.ts index 1f6793fe2..58e23ff5d 100644 --- a/scripts/performance/app-isolation.test.ts +++ b/scripts/performance/app-isolation.test.ts @@ -22,7 +22,15 @@ test('the benchmark app has no Harness or example UI dependencies', async () => }) test('performance CI builds the standalone Release app without custom variants', async () => { - const workflow = await read('.github/workflows/performance.yml') + const workflow = ( + await Promise.all( + [ + '.github/workflows/performance.yml', + 'scripts/performance/build-android.sh', + 'scripts/performance/build-ios.sh', + ].map(read) + ) + ).join('\n') expect(workflow).toContain('cd apps/benchmark/android') expect(workflow).toContain(':app:assembleRelease') expect(workflow).toContain('-configuration Release') diff --git a/scripts/performance/build-android.sh b/scripts/performance/build-android.sh new file mode 100755 index 000000000..7431d467e --- /dev/null +++ b/scripts/performance/build-android.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "${1:?Usage: build-android.sh }" +# Keep both revisions on the NDK version recorded by this CI testbed. +grep -Fq 'ndkVersion = "29.0.14206865"' apps/benchmark/android/build.gradle +bun install --frozen-lockfile +cd apps/benchmark/android +./gradlew :app:assembleRelease --no-daemon --build-cache -PreactNativeArchitectures=x86_64 diff --git a/scripts/performance/build-artifacts.test.ts b/scripts/performance/build-artifacts.test.ts new file mode 100644 index 000000000..ad006c16e --- /dev/null +++ b/scripts/performance/build-artifacts.test.ts @@ -0,0 +1,129 @@ +import { expect, test } from 'bun:test' +import { + chmod, + mkdtemp, + mkdir, + readFile, + readlink, + rm, + stat, +} from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +// Execute the real packaging/selection steps with tiny stand-ins for native +// compilers. This checks missing-base handling and tar permissions/symlinks. +for (const platform of ['android', 'ios'] as const) { + test.each(['paired', 'same-sha', 'changed-suite'])( + `${platform} app artifacts preserve exact build selection: %s`, + async (mode) => { + const root = await mkdtemp(path.join(os.tmpdir(), 'nitro-app-artifacts-')) + try { + const workflow = Bun.YAML.parse( + await readFile( + new URL('../../.github/workflows/performance.yml', import.meta.url), + 'utf8' + ) + ) as any + const command = workflow.jobs[`build-${platform}`].steps.find( + (step: any) => step.name?.startsWith('Build and package') + ).run + await mkdir(path.join(root, 'head/scripts/performance'), { + recursive: true, + }) + await mkdir(path.join(root, 'bin')) + const bun = path.join(root, 'bin/bun') + await Bun.write( + bun, + '#!/bin/sh\ncp "$GITHUB_WORKSPACE/metadata.json" apps/build.json\n' + ) + await chmod(bun, 0o755) + for (const tool of ['java', 'xcodebuild']) { + const file = path.join(root, 'bin', tool) + await Bun.write(file, '#!/bin/sh\necho fixture-tool-version\n') + await chmod(file, 0o755) + } + const script = path.join( + root, + `head/scripts/performance/build-${platform}.sh` + ) + await Bun.write( + script, + `#!/bin/bash +set -eu +echo "$(basename "$1")" >> "$GITHUB_WORKSPACE/builds" +if [[ '${platform}' == android ]]; then + mkdir -p "$1/apps/benchmark/android/app/build/outputs/apk/release" + echo "$(basename "$1")" > "$1/apps/benchmark/android/app/build/outputs/apk/release/app-release.apk" +else + APP="$1/apps/benchmark/ios/build-benchmark/Build/Products/Release-iphonesimulator/NitroBenchmark.app" + mkdir -p "$APP" + echo "$(basename "$1")" > "$APP/NitroBenchmark" + chmod 755 "$APP/NitroBenchmark" + ln -s NitroBenchmark "$APP/executable-link" +fi +` + ) + await Bun.write( + path.join(root, 'metadata.json'), + JSON.stringify({ + baseSha: 'a', + headSha: mode === 'same-sha' ? 'a' : 'b', + baseSuiteHash: 'c', + headSuiteHash: mode === 'changed-suite' ? 'd' : 'c', + }) + ) + const child = Bun.spawn(['bash', '-euo', 'pipefail', '-c', command], { + cwd: root, + env: { + ...process.env, + GITHUB_WORKSPACE: root, + PATH: `${root}/bin:${process.env.PATH}`, + }, + stdout: 'pipe', + stderr: 'pipe', + }) + const [exitCode, stderr] = await Promise.all([ + child.exited, + new Response(child.stderr).text(), + ]) + expect({ exitCode, error: exitCode === 0 ? '' : stderr }).toEqual({ + exitCode: 0, + error: '', + }) + expect( + (await Bun.file(path.join(root, 'builds')).text()).trim().split('\n') + ).toEqual(mode === 'paired' ? ['head', 'base'] : ['head']) + const base = path.join( + root, + `apps/base.${platform === 'ios' ? 'app.tar.gz' : 'apk'}` + ) + expect(await Bun.file(base).exists()).toBe(mode !== 'changed-suite') + if (platform === 'ios') { + const unpack = path.join(root, 'unpacked') + await mkdir(unpack) + expect( + await Bun.spawn([ + 'tar', + '-xzf', + path.join(root, 'apps/head.app.tar.gz'), + '-C', + unpack, + ]).exited + ).toBe(0) + expect( + (await stat(path.join(unpack, 'NitroBenchmark.app/NitroBenchmark'))) + .mode & 0o111 + ).toBe(0o111) + expect( + await readlink( + path.join(unpack, 'NitroBenchmark.app/executable-link') + ) + ).toBe('NitroBenchmark') + } + } finally { + await rm(root, { recursive: true, force: true }) + } + } + ) +} diff --git a/scripts/performance/build-ios.sh b/scripts/performance/build-ios.sh new file mode 100755 index 000000000..8a98e1b6d --- /dev/null +++ b/scripts/performance/build-ios.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "${1:?Usage: build-ios.sh }" +bun install --frozen-lockfile +cd apps/benchmark +bundle install +bun pods +cd ios +xcodebuild \ + CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ \ + -derivedDataPath build-benchmark \ + -workspace NitroBenchmark.xcworkspace \ + -scheme NitroBenchmark \ + -configuration Release \ + -sdk iphonesimulator \ + -destination 'generic/platform=iOS Simulator' \ + ARCHS=arm64 \ + ONLY_ACTIVE_ARCH=YES \ + CODE_SIGNING_ALLOWED=NO \ + COMPILER_INDEX_STORE_ENABLE=NO \ + build diff --git a/scripts/performance/build-metadata.ts b/scripts/performance/build-metadata.ts new file mode 100644 index 000000000..f85418426 --- /dev/null +++ b/scripts/performance/build-metadata.ts @@ -0,0 +1,50 @@ +import { parseArguments, requiredArgument } from './args' +import { calculateSuiteHash } from './suite-hash' + +/** Stored beside the exact apps, and copied unchanged into raw results. */ +export interface BuildMetadata { + platform: 'android' | 'ios' + baseSha: string + headSha: string + baseSuiteHash: string + headSuiteHash: string + architecture: string + toolchain: string + configuration: 'Release' + workflowRunId: number + runAttempt: number +} + +if (import.meta.main) { + const args = parseArguments(Bun.argv.slice(2)) + const platform = requiredArgument(args, 'platform') + if (platform !== 'android' && platform !== 'ios') + throw new Error('Invalid platform.') + const baseRoot = requiredArgument(args, 'base-root') + const headRoot = requiredArgument(args, 'head-root') + async function revision(root: string): Promise { + const child = Bun.spawn(['git', '-C', root, 'rev-parse', 'HEAD'], { + stdout: 'pipe', + }) + const sha = (await new Response(child.stdout).text()).trim() + if ((await child.exited) !== 0) + throw new Error('Cannot resolve built revision.') + return sha + } + const metadata: BuildMetadata = { + platform, + baseSha: await revision(baseRoot), + headSha: await revision(headRoot), + baseSuiteHash: await calculateSuiteHash(baseRoot), + headSuiteHash: await calculateSuiteHash(headRoot), + architecture: requiredArgument(args, 'architecture'), + toolchain: requiredArgument(args, 'toolchain'), + configuration: 'Release', + workflowRunId: Number(process.env.GITHUB_RUN_ID), + runAttempt: Number(process.env.GITHUB_RUN_ATTEMPT), + } + await Bun.write( + requiredArgument(args, 'output'), + `${JSON.stringify(metadata, null, 2)}\n` + ) +} diff --git a/scripts/performance/publish.test.ts b/scripts/performance/publish.test.ts index bb57fec14..99f1a326e 100644 --- a/scripts/performance/publish.test.ts +++ b/scripts/performance/publish.test.ts @@ -62,6 +62,9 @@ describe('Bencher publications', () => { ]) for (const { command } of publications) { expect(command).not.toContain('--start-point-reset') + expect(command).not.toContain('--github-actions') + expect(command).not.toContain('--ci-number') + expect(command).not.toContain('--error-on-alert') } }) diff --git a/scripts/performance/publish.ts b/scripts/performance/publish.ts index 636e85357..f1b4c5673 100644 --- a/scripts/performance/publish.ts +++ b/scripts/performance/publish.ts @@ -84,11 +84,8 @@ if (import.meta.main) { const directory = requiredArgument(argumentsMap, 'directory') const project = process.env.BENCHER_PROJECT const apiKey = process.env.BENCHER_API_KEY - const githubToken = process.env.GITHUB_TOKEN - if (!project?.trim() || !apiKey?.trim() || !githubToken?.trim()) { - throw new Error( - 'BENCHER_PROJECT, BENCHER_API_KEY, and GITHUB_TOKEN are required.' - ) + if (!project?.trim() || !apiKey?.trim()) { + throw new Error('BENCHER_PROJECT and BENCHER_API_KEY are required.') } const metadata: ReportMetadata = JSON.parse( await readFile(path.join(directory, 'metadata.json'), 'utf8') @@ -98,17 +95,6 @@ if (import.meta.main) { directory, project )) { - if (revision === 'head') { - command.push( - '--github-actions', - githubToken, - '--ci-id', - `nitro-${platform}-release`, - '--ci-public-links' - ) - if (metadata.pullRequestNumber != null) - command.push('--ci-number', String(metadata.pullRequestNumber)) - } // The Bencher key is environment-only, never an argument or log message. const child = Bun.spawn(command, { env: { ...Bun.env, BENCHER_API_KEY: apiKey }, diff --git a/scripts/performance/report-markdown.ts b/scripts/performance/report-markdown.ts index ccaaf0e4f..0e434bb7e 100644 --- a/scripts/performance/report-markdown.ts +++ b/scripts/performance/report-markdown.ts @@ -3,6 +3,7 @@ import { type MetricComparison, type PlatformComparison, } from './comparison' +import type { PerformanceReport } from './report' const OPERATION_NAMES: Readonly> = { 'add-numbers': 'addNumbers()', @@ -117,6 +118,7 @@ export function renderPerformanceReportMarkdown( workflowRunUrl?: string artifactId?: number runAttempt?: number + artifacts?: PerformanceReport['artifacts'] } ): string { const lines = [ @@ -181,5 +183,15 @@ export function renderPerformanceReportMarkdown( '' ) } + const platformArtifacts = options.artifacts + if (platformArtifacts != null && options.workflowRunUrl != null) { + lines.push( + ...(['android', 'ios'] as const).map((platform) => { + const artifacts = platformArtifacts[platform] + return `${platformName(platform)}: [measurements, attempt ${artifacts.measurementAttempt}](${options.workflowRunUrl}/artifacts/${artifacts.measurementId}), [apps, attempt ${artifacts.buildAttempt}](${options.workflowRunUrl}/artifacts/${artifacts.buildId}).` + }), + '' + ) + } return lines.join('\n') } diff --git a/scripts/performance/report-validation.test.ts b/scripts/performance/report-validation.test.ts index 1d516da1c..121137cf6 100644 --- a/scripts/performance/report-validation.test.ts +++ b/scripts/performance/report-validation.test.ts @@ -72,12 +72,29 @@ async function createFixture(root: string): Promise<{ output: string trustedEvent: string trustedPullRequest: string + trustedArtifacts: string }> { const artifact = path.join(root, 'artifact') const output = path.join(root, 'output') for (const platform of ['android', 'ios'] as const) { const directory = path.join(artifact, 'raw', platform) await mkdir(directory, { recursive: true }) + await writeJson(path.join(directory, 'measurement.json'), { + buildArtifactId: platform === 'android' ? 1 : 3, + runAttempt: 1, + }) + await writeJson(path.join(directory, 'build.json'), { + platform, + baseSha: BASE_SHA, + headSha: HEAD_SHA, + baseSuiteHash: SUITE_HASH, + headSuiteHash: SUITE_HASH, + architecture: platform === 'ios' ? 'arm64' : 'x86_64', + toolchain: 'fixed toolchain', + configuration: 'Release', + workflowRunId: 123456789, + runAttempt: 1, + }) for (const revision of ['base', 'head'] as const) { for (const sequence of [1, 2]) { await writeJson( @@ -99,6 +116,20 @@ async function createFixture(root: string): Promise<{ headSuiteHash: SUITE_HASH, workflowRunId: 123456789, runAttempt: 1, + artifacts: { + android: { + buildId: 1, + buildAttempt: 1, + measurementId: 2, + measurementAttempt: 1, + }, + ios: { + buildId: 3, + buildAttempt: 1, + measurementId: 4, + measurementAttempt: 1, + }, + }, }) const trustedEvent = path.join(root, 'workflow-run.json') @@ -120,7 +151,20 @@ async function createFixture(root: string): Promise<{ base: { sha: BASE_SHA, repo: { full_name: REPOSITORY } }, head: { sha: HEAD_SHA, repo: { full_name: FORK_REPOSITORY } }, }) - return { artifact, output, trustedEvent, trustedPullRequest } + const trustedArtifacts = path.join(root, 'trusted-artifacts.json') + await writeJson(trustedArtifacts, [ + { id: 1, name: 'performance-apps-android-1', expired: false }, + { id: 2, name: 'performance-android-1', expired: false }, + { id: 3, name: 'performance-apps-ios-1', expired: false }, + { id: 4, name: 'performance-ios-1', expired: false }, + ]) + return { + artifact, + output, + trustedEvent, + trustedPullRequest, + trustedArtifacts, + } } async function validate(fixture: Awaited>) { @@ -136,6 +180,8 @@ async function validate(fixture: Awaited>) { REPOSITORY, '--trusted-workflow-event', fixture.trustedEvent, + '--trusted-artifacts', + fixture.trustedArtifacts, '--artifact-id', '987', '--trusted-pull-request', @@ -269,6 +315,17 @@ describe('trusted performance report validation', () => { const file = path.join(fixture.artifact, 'performance-report.json') const manifest = JSON.parse(await readFile(file, 'utf8')) manifest.baseSuiteHash = 'd'.repeat(64) + for (const platform of ['android', 'ios']) { + const buildFile = path.join( + fixture.artifact, + 'raw', + platform, + 'build.json' + ) + const build = await Bun.file(buildFile).json() + build.baseSuiteHash = manifest.baseSuiteHash + await writeJson(buildFile, build) + } await writeJson(file, manifest) expect((await validate(fixture)).exitCode).toBe(0) expect( @@ -282,8 +339,48 @@ describe('trusted performance report validation', () => { path.join(fixture.output, 'bencher-base-ios.json') ).exists() ).toBe(false) + const headFile = path.join(fixture.artifact, 'raw/ios/head-2.json') + const head = await Bun.file(headFile).json() + head.metrics[0].iterations += 1 + await writeJson(headFile, head) + expect((await validate(fixture)).error).toContain('unequal work') } finally { await rm(root, { recursive: true, force: true }) } }) }) + +test('rerunning Android retains the exact earlier iOS measurements and app builds', async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'nitro-attempts-')) + try { + const fixture = await createFixture(root) + const manifestFile = path.join(fixture.artifact, 'performance-report.json') + const manifest = await Bun.file(manifestFile).json() + manifest.runAttempt = 2 + manifest.artifacts.android.measurementId = 5 + manifest.artifacts.android.measurementAttempt = 2 + await writeJson( + path.join(fixture.artifact, 'raw/android/measurement.json'), + { buildArtifactId: 1, runAttempt: 2 } + ) + await writeJson(manifestFile, manifest) + const event = await Bun.file(fixture.trustedEvent).json() + event.workflow_run.run_attempt = 2 + await writeJson(fixture.trustedEvent, event) + const artifacts = await Bun.file(fixture.trustedArtifacts).json() + artifacts.push({ id: 5, name: 'performance-android-2', expired: false }) + await writeJson(fixture.trustedArtifacts, artifacts) + expect(await validate(fixture)).toEqual({ exitCode: 0, error: '' }) + const markdown = await Bun.file( + path.join(fixture.output, 'performance-summary.md') + ).text() + expect(markdown).toContain('Android: [measurements, attempt 2]') + expect(markdown).toContain('iOS: [measurements, attempt 1]') + // An unrelated artifact cannot substitute for the retained iOS attempt. + manifest.artifacts.ios.measurementId = 5 + await writeJson(manifestFile, manifest) + expect((await validate(fixture)).exitCode).not.toBe(0) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/scripts/performance/report.ts b/scripts/performance/report.ts index 44153275e..f536e37f0 100644 --- a/scripts/performance/report.ts +++ b/scripts/performance/report.ts @@ -1,3 +1,4 @@ +import path from 'node:path' import { readFile } from 'node:fs/promises' import { parseArguments, requiredArgument } from './args' @@ -13,11 +14,19 @@ export interface PerformanceReport { headSuiteHash: string workflowRunId: number runAttempt: number + artifacts: Record<'android' | 'ios', PlatformArtifacts> +} + +export interface PlatformArtifacts { + buildId: number + buildAttempt: number + measurementId: number + measurementAttempt: number } export interface ReportMetadata extends Omit< PerformanceReport, - 'schemaVersion' | 'workflowRunId' | 'runAttempt' + 'schemaVersion' | 'workflowRunId' | 'runAttempt' | 'artifacts' > { platforms: ('android' | 'ios')[] } @@ -36,7 +45,28 @@ if (import.meta.main) { const { baseSuiteHash, headSuiteHash } = JSON.parse( await readFile(requiredArgument(args, 'suite'), 'utf8') ) as Pick + const raw = requiredArgument(args, 'raw-directory') + async function artifacts( + platform: 'android' | 'ios' + ): Promise { + const build = await Bun.file(path.join(raw, platform, 'build.json')).json() + const measurement = await Bun.file( + path.join(raw, platform, 'measurement.json') + ).json() + return { + buildId: measurement.buildArtifactId, + buildAttempt: build.runAttempt, + measurementId: Number( + process.env[`${platform.toUpperCase()}_ARTIFACT_ID`] + ), + measurementAttempt: measurement.runAttempt, + } + } const report: PerformanceReport = { + artifacts: { + android: await artifacts('android'), + ios: await artifacts('ios'), + }, baseSuiteHash, headSuiteHash, schemaVersion: 2, diff --git a/scripts/performance/run-sequence.test.ts b/scripts/performance/run-sequence.test.ts index 7d617675c..263029ec7 100644 --- a/scripts/performance/run-sequence.test.ts +++ b/scripts/performance/run-sequence.test.ts @@ -2,12 +2,18 @@ import { expect, test } from 'bun:test' import { chmod, mkdir, mkdtemp, readFile, rm } from 'node:fs/promises' import os from 'node:os' import path from 'node:path' +import { calculateSuiteHash } from './suite-hash' // Exercise the real controller/receiver with a tiny process standing in for // simctl's app. Runner tests separately exercise timed work and slowdown bounds. -test.each([false, true])( - 'fresh processes share calibrated work; changed suite = %s', - async (changedSuite) => { +test.each([ + [false, false], + [true, false], + [false, true], + [true, true], +])( + 'fresh processes share calibrated work; changed suite = %s, saved apps = %s', + async (changedSuite, savedApps) => { const directory = await mkdtemp(path.join(os.tmpdir(), 'nitro-sequence-')) try { const simulator = path.join(directory, 'simulator.ts') @@ -54,6 +60,24 @@ test.each([false, true])( '// old benchmark' ) } + const metadataPath = path.join(directory, 'build.json') + await Bun.write( + metadataPath, + JSON.stringify({ + platform: 'ios', + baseSha: 'a'.repeat(40), + headSha: 'b'.repeat(40), + baseSuiteHash: await calculateSuiteHash( + changedSuite ? baseRoot : root + ), + headSuiteHash: await calculateSuiteHash(root), + architecture: 'arm64', + toolchain: 'fixture', + configuration: 'Release', + workflowRunId: 123, + runAttempt: 1, + }) + ) const child = Bun.spawn( [ 'bun', @@ -64,10 +88,14 @@ test.each([false, true])( directory, '--head-app', directory, - '--base-root', - changedSuite ? baseRoot : root, - '--head-root', - root, + ...(savedApps + ? ['--build-metadata', metadataPath] + : [ + '--base-root', + changedSuite ? baseRoot : root, + '--head-root', + root, + ]), '--base-sha', 'a'.repeat(40), '--head-sha', @@ -90,6 +118,9 @@ test.each([false, true])( ...process.env, PATH: `${directory}:${process.env.PATH}`, SIMULATOR_LOG: log, + GITHUB_RUN_ID: '123', + GITHUB_RUN_ATTEMPT: '2', + BUILD_ARTIFACT_ID: '456', CHANGED_SUITE: String(changedSuite), }, stdout: 'pipe', @@ -108,6 +139,14 @@ test.each([false, true])( exitCode: 0, error: '', }) + if (savedApps) { + expect( + (await Bun.file(path.join(output, 'build.json')).json()).runAttempt + ).toBe(1) + expect( + await Bun.file(path.join(output, 'measurement.json')).json() + ).toEqual({ buildArtifactId: 456, runAttempt: 2 }) + } const processes = (await readFile(log, 'utf8')) .trim() .split('\n') diff --git a/scripts/performance/run-sequence.ts b/scripts/performance/run-sequence.ts index dabdc8bde..6d9151b82 100644 --- a/scripts/performance/run-sequence.ts +++ b/scripts/performance/run-sequence.ts @@ -3,6 +3,7 @@ import path from 'node:path' import { parseArguments, requiredArgument } from './args' import { validateBenchmarkRun } from './schema' import { calculateSuiteHash } from './suite-hash' +import type { BuildMetadata } from './build-metadata' const argumentsMap = parseArguments(Bun.argv.slice(2)) const platform = requiredArgument(argumentsMap, 'platform') @@ -11,8 +12,6 @@ if (platform !== 'android' && platform !== 'ios') { } const baseApp = path.resolve(requiredArgument(argumentsMap, 'base-app')) const headApp = path.resolve(requiredArgument(argumentsMap, 'head-app')) -const baseRoot = path.resolve(requiredArgument(argumentsMap, 'base-root')) -const headRoot = path.resolve(requiredArgument(argumentsMap, 'head-root')) const baseSha = requiredArgument(argumentsMap, 'base-sha') const headSha = requiredArgument(argumentsMap, 'head-sha') const outputDirectory = path.resolve( @@ -25,10 +24,48 @@ const architecture = requiredArgument(argumentsMap, 'architecture') const toolchain = requiredArgument(argumentsMap, 'toolchain') await mkdir(outputDirectory, { recursive: true }) -const [baseSuiteHash, headSuiteHash] = await Promise.all([ - calculateSuiteHash(baseRoot), - calculateSuiteHash(headRoot), -]) +// CI binds the downloaded apps to their original build, even on job reruns. +// Local callers can still point at their two source checkouts. +const metadataPath = argumentsMap.get('build-metadata')?.[0] +const build: BuildMetadata | undefined = + metadataPath == null + ? undefined + : JSON.parse(await readFile(metadataPath, 'utf8')) +if ( + build != null && + (build.baseSha !== baseSha || + build.headSha !== headSha || + build.platform !== platform || + build.architecture !== architecture || + build.toolchain !== toolchain || + build.configuration !== 'Release' || + build.workflowRunId !== Number(process.env.GITHUB_RUN_ID)) +) { + throw new Error( + 'Downloaded app metadata does not match the requested revisions or testbed.' + ) +} +const [baseSuiteHash, headSuiteHash] = + build == null + ? await Promise.all([ + calculateSuiteHash( + path.resolve(requiredArgument(argumentsMap, 'base-root')) + ), + calculateSuiteHash( + path.resolve(requiredArgument(argumentsMap, 'head-root')) + ), + ]) + : [build.baseSuiteHash, build.headSuiteHash] +if (build != null) { + await Bun.write( + path.join(outputDirectory, 'build.json'), + `${JSON.stringify(build, null, 2)}\n` + ) + await Bun.write( + path.join(outputDirectory, 'measurement.json'), + `${JSON.stringify({ buildArtifactId: Number(process.env.BUILD_ARTIFACT_ID), runAttempt: Number(process.env.GITHUB_RUN_ATTEMPT) }, null, 2)}\n` + ) +} await Bun.write( path.join(outputDirectory, 'suite.json'), diff --git a/scripts/performance/select-report.ts b/scripts/performance/select-report.ts index 68392504f..45222438d 100644 --- a/scripts/performance/select-report.ts +++ b/scripts/performance/select-report.ts @@ -1,6 +1,6 @@ import { appendFile, readFile } from 'node:fs/promises' -interface Artifact { +export interface Artifact { id: number name: string expired: boolean @@ -69,6 +69,10 @@ if (import.meta.main) { ]) if (artifactResponse.total_count > 100 || jobResponse.total_count > 100) throw new Error('Performance run exceeds the artifact/job lookup limit.') + await Bun.write( + 'trusted-artifacts.json', + JSON.stringify(artifactResponse.artifacts) + ) const id = selectReportArtifact( run.conclusion, run.run_attempt, diff --git a/scripts/performance/validate-report.ts b/scripts/performance/validate-report.ts index 0f6e96e11..7c1436924 100644 --- a/scripts/performance/validate-report.ts +++ b/scripts/performance/validate-report.ts @@ -1,4 +1,5 @@ -import type { PerformanceReport } from './report' +import type { PerformanceReport, PlatformArtifacts } from './report' +import type { Artifact } from './select-report' import { appendFile, mkdir, readdir, readFile, stat } from 'node:fs/promises' import path from 'node:path' import { parseArguments, requiredArgument } from './args' @@ -194,7 +195,30 @@ function validateReport(value: unknown): PerformanceReport { !/^[0-9a-f]{64}$/.test(headSuiteHash) ) throw new Error('Invalid report suite hashes.') + const artifactValues = object(report.artifacts, 'report.artifacts') + function artifacts(platform: 'android' | 'ios'): PlatformArtifacts { + const values = object(artifactValues[platform], `${platform} artifacts`) + const result = {} as PlatformArtifacts + for (const key of [ + 'buildId', + 'buildAttempt', + 'measurementId', + 'measurementAttempt', + ] as const) { + const value = finiteNumber(values[key], `${platform}.${key}`) + if (!Number.isSafeInteger(value) || value < 1) + throw new Error('Invalid platform artifact provenance.') + result[key] = value + } + if ( + result.buildAttempt > result.measurementAttempt || + result.measurementAttempt > Number(report.runAttempt) + ) + throw new Error('Platform artifacts come from a future attempt.') + return result + } return { + artifacts: { android: artifacts('android'), ios: artifacts('ios') }, baseSuiteHash, headSuiteHash, schemaVersion: 2, @@ -277,6 +301,63 @@ if (report.eventName === 'pull_request') { ) } } +const builds = new Map>() +const trustedArtifacts = (await readBoundedJson( + requiredArgument(argumentsMap, 'trusted-artifacts') +)) as Artifact[] +for (const platform of ['android', 'ios'] as const) { + const ids = report.artifacts[platform] + for (const [id, name] of [ + [ids.buildId, `performance-apps-${platform}-${ids.buildAttempt}`], + [ids.measurementId, `performance-${platform}-${ids.measurementAttempt}`], + ] as const) { + if ( + !trustedArtifacts.some( + (artifact) => + artifact.id === id && artifact.name === name && !artifact.expired + ) + ) { + throw new Error( + `${platform} references an artifact outside this workflow run or attempt.` + ) + } + } + const build = object( + await readBoundedJson( + path.join(artifactDirectory, 'raw', platform, 'build.json') + ), + `${platform} build` + ) + if ( + build.platform !== platform || + build.baseSha !== report.baseSha || + build.headSha !== report.headSha || + build.baseSuiteHash !== report.baseSuiteHash || + build.headSuiteHash !== report.headSuiteHash || + build.workflowRunId !== report.workflowRunId || + build.runAttempt !== ids.buildAttempt || + build.configuration !== 'Release' + ) { + throw new Error( + `${platform} app build metadata does not match this report.` + ) + } + const measurement = object( + await readBoundedJson( + path.join(artifactDirectory, 'raw', platform, 'measurement.json') + ), + `${platform} measurement` + ) + if ( + measurement.buildArtifactId !== ids.buildId || + measurement.runAttempt !== ids.measurementAttempt + ) + throw new Error( + `${platform} measurement provenance does not match this report.` + ) + builds.set(platform, build) +} + async function loadRawRuns( platform: 'android' | 'ios', revision: 'base' | 'head', @@ -300,6 +381,8 @@ async function loadRawRuns( run.configuration.runId !== `${platform}-${revision}-${sequence}` || run.configuration.reverse !== (sequence === 2) || run.configuration.platform !== platform || + run.configuration.architecture !== builds.get(platform)!.architecture || + run.configuration.toolchain !== builds.get(platform)!.toolchain || run.configuration.benchmarkIndex !== undefined || run.metrics.length !== run.benchmarkCount || run.configuration.commitSha !== expectedSha || @@ -325,6 +408,8 @@ async function loadRawRuns( const rebuiltComparisons = await Promise.all( (['android', 'ios'] as const).map(async (platform) => { const headRuns = await loadRawRuns(platform, 'head', report.headSha) + // This also checks the two baseline processes share work and runtime settings. + compareRuns([headRuns[0]!], [headRuns[1]!]) const comparable = report.baseSuiteHash === report.headSuiteHash const baseFiles = ( await readdir(path.join(artifactDirectory, 'raw', platform)) @@ -359,9 +444,11 @@ const markdown = renderPerformanceReportMarkdown( workflowRunUrl, artifactId: Number(requiredArgument(argumentsMap, 'artifact-id')), runAttempt: report.runAttempt, + artifacts: report.artifacts, } ) await Bun.write(path.join(outputDirectory, 'performance-summary.md'), markdown) + await Bun.write( path.join(outputDirectory, 'metadata.json'), `${JSON.stringify( diff --git a/scripts/performance/workflow.test.ts b/scripts/performance/workflow.test.ts index aff4d45ea..efba6cbcd 100644 --- a/scripts/performance/workflow.test.ts +++ b/scripts/performance/workflow.test.ts @@ -1,5 +1,7 @@ import { expect, test } from 'bun:test' -import { readFile } from 'node:fs/promises' +import { readFile, mkdtemp, mkdir, rm } from 'node:fs/promises' +import path from 'node:path' +import os from 'node:os' test('Android performance CI requires KVM and cannot fall back to software emulation', async () => { const source = await readFile( @@ -8,7 +10,7 @@ test('Android performance CI requires KVM and cannot fall back to software emula ) const workflow = Bun.YAML.parse(source) as { jobs: { - android: { + 'measure-android': { steps: { name?: string run?: string @@ -17,12 +19,12 @@ test('Android performance CI requires KVM and cannot fall back to software emula } } } - const steps = workflow.jobs.android.steps + const steps = workflow.jobs['measure-android'].steps const kvmIndex = steps.findIndex( (step) => step.name === 'Enable KVM for benchmark measurements' ) const buildIndex = steps.findIndex( - (step) => step.name === 'Build head benchmark APK' + (step) => step.name === 'Run paired Android benchmarks' ) expect(kvmIndex).toBeGreaterThanOrEqual(0) expect(kvmIndex).toBeLessThan(buildIndex) @@ -78,7 +80,82 @@ test('one trusted publisher handles internal and fork reports without executing ).toBe('${{ steps.select.outputs.artifact_id }}') expect( steps.findIndex((s) => s.name === 'Verify pinned Bencher binary') - ).toBeLessThan( - steps.findIndex((s) => s.name === 'Publish to Bencher and GitHub') - ) + ).toBeLessThan(steps.findIndex((s) => s.name === 'Publish Bencher history')) }) + +test.each([false, true])( + 'package docs skip measurements unless native code also changed: %s', + async (nativeChange) => { + const root = await mkdtemp(path.join(os.tmpdir(), 'nitro-relevance-')) + try { + async function git(...args: string[]) { + const child = Bun.spawn(['git', ...args], { + cwd: root, + stdout: 'pipe', + stderr: 'pipe', + }) + const output = await new Response(child.stdout).text() + if ((await child.exited) !== 0) + throw new Error(await new Response(child.stderr).text()) + return output.trim() + } + await git('init', '-q') + await git( + '-c', + 'user.name=Fixture', + '-c', + 'user.email=fixture@example.com', + 'commit', + '--allow-empty', + '-qm', + 'base' + ) + const base = await git('rev-parse', 'HEAD') + const directory = path.join(root, 'packages/react-native-nitro-modules') + await mkdir(directory, { recursive: true }) + await Bun.write(path.join(directory, 'README.md'), 'documentation') + await Bun.write(path.join(directory, 'guide.mdx'), 'documentation') + if (nativeChange) + await Bun.write(path.join(directory, 'Runtime.cpp'), '// native change') + await git('add', '.') + await git( + '-c', + 'user.name=Fixture', + '-c', + 'user.email=fixture@example.com', + 'commit', + '-qm', + 'head' + ) + const workflow = Bun.YAML.parse( + await readFile( + new URL('../../.github/workflows/performance.yml', import.meta.url), + 'utf8' + ) + ) as any + const script = workflow.jobs.prepare.steps.find( + (step: any) => step.id === 'metadata' + ).run + const child = Bun.spawn(['bash', '-euo', 'pipefail', '-c', script], { + cwd: root, + env: { + ...process.env, + EVENT_NAME: 'pull_request', + PR_BASE_SHA: base, + PR_HEAD_SHA: await git('rev-parse', 'HEAD'), + PR_NUMBER: '1', + GITHUB_OUTPUT: path.join(root, 'outputs'), + GITHUB_STEP_SUMMARY: path.join(root, 'summary'), + }, + stdout: 'pipe', + stderr: 'pipe', + }) + expect(await child.exited).toBe(0) + expect(await Bun.file(path.join(root, 'outputs')).text()).toContain( + `relevant=${nativeChange}` + ) + } finally { + await rm(root, { recursive: true, force: true }) + } + } +)