Test262 Coverage #1879
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test262 Coverage | |
| on: | |
| workflow_run: | |
| workflows: ["CI"] | |
| types: [completed] | |
| workflow_dispatch: | |
| inputs: | |
| ci_run_id: | |
| description: "Optional CI run id to reuse qjs-cli-release-linux artifact" | |
| required: false | |
| type: string | |
| # One coverage run per branch: a newer successful CI run supersedes an | |
| # in-flight run for the same branch instead of queueing a full 32-shard scan | |
| # per commit. Failed or cancelled workflow_run events use a unique group so | |
| # they cannot cancel useful coverage that they would not replace. The main CI | |
| # workflow already cancels superseded runs per branch, so rapid pushes never | |
| # had a complete per-commit burndown guarantee; the burndown trend in | |
| # docs/conformance/burndown.jsonl only needs the latest state per branch. | |
| concurrency: | |
| group: >- | |
| ${{ (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success') | |
| && format('test262-coverage-{0}', github.event.workflow_run.head_branch || github.ref_name) | |
| || format('test262-coverage-skip-{0}', github.run_id) }} | |
| cancel-in-progress: true | |
| jobs: | |
| quickjs-ng-baseline-cache: | |
| if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' | |
| name: QuickJS-NG baseline cache | |
| runs-on: ubuntu-latest | |
| outputs: | |
| cache-hit: ${{ steps.quickjs-ng-baseline-cache.outputs.cache-hit }} | |
| cache-key: ${{ steps.baseline-key.outputs.full_key }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ github.event.workflow_run.head_repository.full_name || github.repository }} | |
| ref: ${{ github.event.workflow_run.head_sha || github.sha }} | |
| fetch-depth: 1 | |
| submodules: false | |
| - name: Baseline cache key | |
| id: baseline-key | |
| run: | | |
| echo "quickjs_ng_sha=$(git rev-parse HEAD:third_party/quickjs-ng)" >> "$GITHUB_OUTPUT" | |
| echo "test262_sha=$(git rev-parse HEAD:third_party/test262)" >> "$GITHUB_OUTPUT" | |
| echo "full_key=test262-qjsng-baseline-full-${{ runner.os }}-$(git rev-parse HEAD:third_party/quickjs-ng)-$(git rev-parse HEAD:third_party/test262)-${{ hashFiles('scripts/test262-baseline.sh', 'scripts/test262-baseline-metadata.awk', 'scripts/run-with-timeout.sh') }}" >> "$GITHUB_OUTPUT" | |
| - name: Restore QuickJS-NG baseline | |
| id: quickjs-ng-baseline-cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: coverage-baseline | |
| key: ${{ steps.baseline-key.outputs.full_key }} | |
| - name: Upload cached QuickJS-NG baseline | |
| if: steps.quickjs-ng-baseline-cache.outputs.cache-hit == 'true' | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test262-qjsng-baseline-full | |
| path: coverage-baseline | |
| retention-days: 14 | |
| quickjs-ng-baseline: | |
| if: | | |
| (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success') && | |
| needs.quickjs-ng-baseline-cache.outputs.cache-hit != 'true' | |
| name: QuickJS-NG baseline shard ${{ matrix.shard }}/16 | |
| needs: quickjs-ng-baseline-cache | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ github.event.workflow_run.head_repository.full_name || github.repository }} | |
| ref: ${{ github.event.workflow_run.head_sha || github.sha }} | |
| fetch-depth: 1 | |
| submodules: false | |
| - name: Baseline cache key | |
| id: baseline-key | |
| run: | | |
| echo "quickjs_ng_sha=$(git rev-parse HEAD:third_party/quickjs-ng)" >> "$GITHUB_OUTPUT" | |
| echo "test262_sha=$(git rev-parse HEAD:third_party/test262)" >> "$GITHUB_OUTPUT" | |
| - name: Restore QuickJS-NG baseline shard | |
| id: quickjs-ng-baseline-shard-cache | |
| uses: actions/cache/restore@v5 | |
| with: | |
| path: coverage-baseline/shard-${{ matrix.shard }} | |
| key: test262-qjsng-baseline-${{ runner.os }}-${{ steps.baseline-key.outputs.quickjs_ng_sha }}-${{ steps.baseline-key.outputs.test262_sha }}-${{ hashFiles('scripts/test262-baseline.sh', 'scripts/test262-baseline-metadata.awk', 'scripts/run-with-timeout.sh') }}-${{ matrix.shard }} | |
| - name: Init submodules | |
| if: steps.quickjs-ng-baseline-shard-cache.outputs.cache-hit != 'true' | |
| run: git submodule update --init --depth 1 third_party/test262 third_party/quickjs-ng | |
| - name: Restore QuickJS-NG build | |
| if: steps.quickjs-ng-baseline-shard-cache.outputs.cache-hit != 'true' | |
| uses: actions/cache@v5 | |
| with: | |
| path: third_party/quickjs-ng/build | |
| key: quickjs-ng-${{ runner.os }}-${{ steps.baseline-key.outputs.quickjs_ng_sha }} | |
| - name: Run QuickJS-NG baseline shard | |
| if: steps.quickjs-ng-baseline-shard-cache.outputs.cache-hit != 'true' | |
| run: | | |
| mkdir -p coverage-baseline/shard-${{ matrix.shard }} | |
| ./scripts/test262-baseline.sh \ | |
| --all \ | |
| --engine quickjs-ng \ | |
| --shard ${{ matrix.shard }}/16 \ | |
| --summary-json coverage-baseline/shard-${{ matrix.shard }}/summary-${{ matrix.shard }}.json \ | |
| --case-results-jsonl coverage-baseline/shard-${{ matrix.shard }}/cases-${{ matrix.shard }}.jsonl \ | |
| --no-fail | |
| - name: Save QuickJS-NG baseline | |
| if: steps.quickjs-ng-baseline-shard-cache.outputs.cache-hit != 'true' | |
| continue-on-error: true | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: coverage-baseline/shard-${{ matrix.shard }} | |
| key: test262-qjsng-baseline-${{ runner.os }}-${{ steps.baseline-key.outputs.quickjs_ng_sha }}-${{ steps.baseline-key.outputs.test262_sha }}-${{ hashFiles('scripts/test262-baseline.sh', 'scripts/test262-baseline-metadata.awk', 'scripts/run-with-timeout.sh') }}-${{ matrix.shard }} | |
| - name: Upload QuickJS-NG baseline shard | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test262-qjsng-baseline-${{ matrix.shard }} | |
| path: coverage-baseline/shard-${{ matrix.shard }} | |
| retention-days: 14 | |
| save-quickjs-ng-baseline-cache: | |
| name: Save QuickJS-NG baseline cache | |
| needs: [quickjs-ng-baseline-cache, quickjs-ng-baseline] | |
| if: | | |
| always() && | |
| needs.quickjs-ng-baseline-cache.result == 'success' && | |
| needs.quickjs-ng-baseline-cache.outputs.cache-hit != 'true' && | |
| needs.quickjs-ng-baseline.result == 'success' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download QuickJS-NG baseline shards | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: test262-qjsng-baseline-* | |
| path: coverage-baseline | |
| merge-multiple: true | |
| - name: Save full QuickJS-NG baseline | |
| continue-on-error: true | |
| uses: actions/cache/save@v5 | |
| with: | |
| path: coverage-baseline | |
| key: ${{ needs.quickjs-ng-baseline-cache.outputs.cache-key }} | |
| coverage-shards: | |
| if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' | |
| name: Test262 rust coverage group ${{ matrix.group }}/16 | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ github.event.workflow_run.head_repository.full_name || github.repository }} | |
| ref: ${{ github.event.workflow_run.head_sha || github.sha }} | |
| fetch-depth: 1 | |
| submodules: false | |
| - name: Download qjs-cli from CI | |
| id: qjs-cli-artifact | |
| if: github.event_name == 'workflow_run' || github.event.inputs.ci_run_id != '' | |
| continue-on-error: true | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: qjs-cli-release-linux | |
| path: prebuilt-qjs | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| repository: ${{ github.event.workflow_run.head_repository.full_name || github.repository }} | |
| run-id: ${{ github.event.workflow_run.id || github.event.inputs.ci_run_id }} | |
| - name: Prepare qjs-cli artifact | |
| if: steps.qjs-cli-artifact.outcome == 'success' | |
| run: chmod +x prebuilt-qjs/qjs | |
| - name: Setup Rust | |
| if: steps.qjs-cli-artifact.outcome != 'success' | |
| uses: ./.github/actions/setup-rust | |
| - name: Init submodules | |
| run: git submodule update --init --depth 1 third_party/test262 | |
| - name: Build quickjs-rust once | |
| if: steps.qjs-cli-artifact.outcome != 'success' | |
| # The `agents` feature backs the $262.agent multi-agent Atomics cases the | |
| # coverage run executes (QJS_AGENTS=1). Matches the CI artifact build. | |
| run: cargo build -q --release -p qjs-cli --features agents | |
| - name: Run full Test262 coverage shard | |
| env: | |
| TEST262_TIMEOUT_RETRIES: 1 | |
| # Run the $262.agent multi-agent Atomics cases for real (OS threads, | |
| # cross-thread SharedArrayBuffer, Atomics.wait/notify) instead of | |
| # reporting them as not-run. The CLI is built with --features agents. | |
| QJS_AGENTS: 1 | |
| # The agents run additionally executes 61 multi-threaded Atomics cases | |
| # in the shared parallel pool; their threads add CPU contention that can | |
| # push an unrelated, already-slow case (general call-overhead bound, not | |
| # agents overhead — measured identical in both builds) past the default | |
| # 10s budget on a 2-core runner. Give the whole run headroom. The real | |
| # call-perf fix is the env-model rewrite tracked separately. | |
| TEST262_CASE_TIMEOUT_SECONDS: 20 | |
| run: | | |
| mkdir -p coverage | |
| status=0 | |
| if [ -x prebuilt-qjs/qjs ]; then | |
| export QJS_CLI_BIN="$PWD/prebuilt-qjs/qjs" | |
| else | |
| target_dir="$(cargo metadata --format-version=1 --no-deps \ | |
| | sed -n 's/.*"target_directory":"\([^"]*\)".*/\1/p' \ | |
| | head -n 1)" | |
| export QJS_CLI_BIN="${target_dir:-$PWD/target}/release/qjs" | |
| fi | |
| # test262-baseline.sh parallelizes case execution internally, so the | |
| # group's two shards run sequentially instead of racing for cores. | |
| first_shard=$(( (${{ matrix.group }} - 1) * 2 + 1 )) | |
| last_shard=$(( first_shard + 1 )) | |
| for shard in $(seq "$first_shard" "$last_shard"); do | |
| ./scripts/test262-baseline.sh \ | |
| --all \ | |
| --engine quickjs-rust \ | |
| --shard "$shard/32" \ | |
| --summary-json "coverage/summary-${shard}.json" \ | |
| --case-results-jsonl "coverage/cases-${shard}.jsonl" \ | |
| --no-fail || status=1 | |
| done | |
| exit "$status" | |
| - name: Upload shard summary | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test262-coverage-${{ matrix.group }} | |
| path: coverage | |
| retention-days: 14 | |
| aggregate: | |
| name: Aggregate Test262 coverage | |
| runs-on: ubuntu-latest | |
| needs: [quickjs-ng-baseline-cache, quickjs-ng-baseline, coverage-shards] | |
| if: | | |
| always() && | |
| (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success') && | |
| needs.quickjs-ng-baseline-cache.result == 'success' && | |
| needs.coverage-shards.result == 'success' && | |
| ( | |
| needs.quickjs-ng-baseline-cache.outputs.cache-hit == 'true' || | |
| needs.quickjs-ng-baseline.result == 'success' | |
| ) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ github.event.workflow_run.head_repository.full_name || github.repository }} | |
| ref: ${{ github.event.workflow_run.head_sha || github.sha }} | |
| fetch-depth: 1 | |
| submodules: false | |
| - name: Download QuickJS-NG baseline | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: test262-qjsng-baseline-* | |
| path: qjsng-baseline | |
| merge-multiple: true | |
| - name: Download rust shard summaries | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: test262-coverage-* | |
| path: rust-coverage | |
| merge-multiple: true | |
| - name: Write coverage summary, burndown entry, and enforce parity gate | |
| env: | |
| COVERAGE_SHA: ${{ github.event.workflow_run.head_sha || github.sha }} | |
| run: | | |
| python3 scripts/test262-aggregate.py \ | |
| --ng-cases 'qjsng-baseline/cases-*.jsonl' \ | |
| --rust-cases 'rust-coverage/cases-*.jsonl' \ | |
| --commit "$COVERAGE_SHA" \ | |
| --summary-out "$GITHUB_STEP_SUMMARY" \ | |
| --burndown-out burndown.json \ | |
| --comparison-cases-out comparison-cases.jsonl \ | |
| --require-complete-parity | |
| - name: Upload burndown entry | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test262-burndown | |
| path: burndown.json | |
| retention-days: 90 | |
| - name: Upload comparison cases | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: test262-comparison-cases | |
| path: comparison-cases.jsonl | |
| retention-days: 14 |