diff --git a/.github/actions/update-ci-report/action.yml b/.github/actions/update-ci-report/action.yml new file mode 100644 index 00000000..e5a8d83e --- /dev/null +++ b/.github/actions/update-ci-report/action.yml @@ -0,0 +1,66 @@ +name: Update CI report +description: Render all available task fragments and update the unified check/comment + +inputs: + run-dir: + description: Per-run CI state directory + required: true + fallback-conclusion: + description: Conclusion when no plan or fragments survived + required: false + default: failure + app-id: + description: GitHub App ID for cross-repository reports + required: false + app-private-key: + description: GitHub App private key for cross-repository reports + required: false + +runs: + using: composite + steps: + - name: Render report snapshot + shell: bash + env: + CI_RUN_DIR: ${{ inputs.run-dir }} + run: | + python3 scripts/ci/report/render.py \ + --plan "$CI_RUN_DIR/plan.json" \ + --fragments "$CI_RUN_DIR/fragments" \ + --fallback-conclusion '${{ inputs.fallback-conclusion }}' \ + --md-out "$CI_RUN_DIR/report/report.md" \ + --json-out "$CI_RUN_DIR/report/report.json" + + - name: Resolve report destination + id: origin + shell: bash + env: + CI_RUN_DIR: ${{ inputs.run-dir }} + run: | + python3 scripts/ci/origin.py token-target \ + --input "$CI_RUN_DIR/origin.json" \ + --allowed-owner "$GITHUB_REPOSITORY_OWNER" \ + --github-output "$GITHUB_OUTPUT" + + - name: Mint report token + id: report-token + if: steps.origin.outputs.external == 'true' + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ inputs.app-id }} + private-key: ${{ inputs.app-private-key }} + owner: ${{ steps.origin.outputs.owner }} + repositories: ${{ steps.origin.outputs.repository }} + permission-checks: write + permission-issues: write + permission-pull-requests: read + + - name: Publish report snapshot + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + CI_RUN_DIR: ${{ inputs.run-dir }} + with: + github-token: ${{ steps.report-token.outputs.token || github.token }} + script: | + const post = require('${{ github.workspace }}/scripts/ci/report/post.js'); + await post({github, context, core}); diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85ec8233..5c8d14eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,189 +2,706 @@ name: Build on: pull_request: + types: [opened, synchronize, reopened, labeled] merge_group: push: - branches: - - main + branches: [main] + repository_dispatch: + types: [wasinix-ci-v1] + workflow_dispatch: + inputs: + command: + description: CI command, for example "build all --at HEAD" + type: string + default: build all --at HEAD + origin: + description: Validated cross-repository command origin JSON + type: string + default: "" + concurrency: + description: Optional caller concurrency key + type: string + default: "" permissions: contents: read actions: write - # in-job report posting on same-repo events (read-only on fork PRs, which - # test-report.yml covers instead) checks: write pull-requests: write concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - # don't cancel main: let cache-warming runs finish (PRs still cancel) - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + group: ${{ github.workflow }}-${{ github.event.client_payload.concurrency || inputs.concurrency || github.ref }} + # Main builds warm the cache. PR and explicitly keyed custom runs supersede old work. + cancel-in-progress: ${{ github.ref != 'refs/heads/main' || inputs.concurrency != '' || github.event.client_payload.concurrency != '' }} + +env: + CI_RUN_DIR: /tmp/wasinix-ci/${{ github.run_id }}-${{ github.run_attempt }} jobs: - build: - name: Build all packages - runs-on: depot-ubuntu-24.04-16 - # high ceiling for a cold cache; warm runs finish in minutes (max is 360) - timeout-minutes: 350 + prepare: + name: Prepare request + runs-on: ubuntu-latest + outputs: + trusted: ${{ steps.normalize.outputs.trusted }} + all: ${{ steps.matrix.outputs.all }} + treefmt: ${{ steps.matrix.outputs.treefmt }} + eval: ${{ steps.matrix.outputs.eval }} + core: ${{ steps.matrix.outputs.core }} + packages: ${{ steps.matrix.outputs.packages }} + python: ${{ steps.matrix.outputs.python }} + jobs: ${{ steps.matrix.outputs.jobs }} + spot: ${{ steps.matrix.outputs.spot }} + compare: ${{ steps.matrix.outputs.compare }} + content: ${{ steps.matrix.outputs.content }} steps: - - name: Checkout + - name: Checkout orchestrator uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Install Nix uses: cachix/install-nix-action@v31.9.1 - with: - extra_nix_config: | - extra-substituters = https://nix-cache.wasix.org - extra-trusted-public-keys = wasinix-1:jvsqbOJGsZxMvg97fuyNCWCc+t2nn6uHB47kQCGNmXI= - trusted-users = root runner - - # Persist nix's flake-input fetch cache (~2 min/run of cloning wasmer + - # its transitive inputs). Keyed on flake.lock so it invalidates when inputs move. - - name: Cache flake inputs - uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 - with: - path: ~/.cache/nix - key: nix-flake-inputs-${{ hashFiles('flake.lock') }} - restore-keys: nix-flake-inputs- - - # The fetch cache above only avoids re-downloading tarballs; the unpacked - # ...-source store paths still substitute from the cache bucket every run - # (~40s). Persist their closure as a file:// binary cache. Exact key - # only: restore-keys would accrete stale paths into every re-save. - - name: Cache flake input closure - id: input-closure - uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 - with: - path: /tmp/input-store - key: nix-input-closure-${{ hashFiles('flake.lock') }} - - # continue-on-error: a corrupt cache entry must not fail CI; the eval - # below just falls back to substitution. - - name: Import flake input closure - if: steps.input-closure.outputs.cache-hit == 'true' - continue-on-error: true - run: nix copy --all --no-check-sigs --from 'file:///tmp/input-store' - - # eval once (attr -> drvPath), diff against the base branch's published - # map to surface what this change rebuilds; ci-build.sh reuses the raw - # eval. Informational: a missing base map skips the diff, an eval - # failure becomes report content, and a script bug must not eat the - # build or the report (the build step fails the job on real breakage) - - name: Rebuild diff - continue-on-error: true + + - name: Normalize request + id: normalize + shell: bash env: - BASE_REF: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} - run: nix run .#scripts.rebuild-diff + CI_COMMAND: ${{ inputs.command }} + CI_ORIGIN: ${{ inputs.origin }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + CONTENT_DIFF: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:content-diff') }} + run: | + mkdir -p "$CI_RUN_DIR" + trusted=true + if [[ "$GITHUB_EVENT_NAME" == repository_dispatch || -n "$CI_ORIGIN" ]]; then + trusted=false + fi + echo "trusted=$trusted" >>"$GITHUB_OUTPUT" + case "$GITHUB_EVENT_NAME" in + repository_dispatch) + python3 scripts/ci/cli.py accept \ + --event "$GITHUB_EVENT_PATH" --out "$CI_RUN_DIR/request.json" + ;; + workflow_dispatch) + if [[ -n "$CI_ORIGIN" ]]; then + python3 scripts/ci/origin.py validate \ + --value "$CI_ORIGIN" --allowed-owner "$GITHUB_REPOSITORY_OWNER" \ + --out "$CI_RUN_DIR/origin.json" + export WASINIX_CI_ORIGIN="$CI_RUN_DIR/origin.json" + fi + python3 scripts/ci/cli.py request \ + --command-string "$CI_COMMAND" --out "$CI_RUN_DIR/request.json" + ;; + pull_request) + content=() + if [[ "$CONTENT_DIFF" == "true" ]]; then + content=(--content-diff) + fi + python3 scripts/ci/cli.py request --out "$CI_RUN_DIR/request.json" -- \ + diff "${content[@]}" \ + build core packages --at "$BASE_SHA" --vs \ + build core packages --at "$GITHUB_SHA" + ;; + merge_group) + python3 scripts/ci/cli.py request --out "$CI_RUN_DIR/request.json" -- \ + diff build all --at "$BASE_SHA" --vs build all --at "$GITHUB_SHA" + ;; + push) + python3 scripts/ci/cli.py request --out "$CI_RUN_DIR/request.json" -- \ + build all --at "$GITHUB_SHA" + ;; + esac + + - name: Materialize cases and task plan + run: nix run .#scripts.ci -- prepare-all --request "$CI_RUN_DIR/request.json" --run-dir "$CI_RUN_DIR" + + - name: Export matrices + id: matrix + shell: bash + run: | + matrix=$(cat "$CI_RUN_DIR/matrix.json") + echo "all=$(jq -c '.' <<<"$matrix")" >>"$GITHUB_OUTPUT" + for task in treefmt eval core packages python jobs spot compare content; do + selected=$(jq -c --arg task "$task" ' + [.[] | select(.[$task]) | . + {enabled: true}] + | if length == 0 then [{id: "disabled", enabled: false}] else . end + ' <<<"$matrix") + echo "$task=$selected" >>"$GITHUB_OUTPUT" + done + + - name: Upload prepared state + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-state-prepare-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + retention-days: 30 - # after the eval (which realizes every input) but before the build, so a - # build failure or timeout can't lose the post-job cache save - - name: Archive flake input closure - if: steps.input-closure.outputs.cache-hit != 'true' + treefmt: + name: Formatting (${{ matrix.id }}) + needs: prepare + if: ${{ always() && needs.prepare.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.prepare.outputs.treefmt || '[{"id":"disabled","enabled":false}]') }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v31.9.1 + - name: Download prepared state + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ci-state-prepare-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + - name: Check formatting + id: formatting + if: matrix.enabled continue-on-error: true - run: nix flake archive --to 'file:///tmp/input-store?compression=zstd' + run: | + nix run .#scripts.ci -- treefmt-task \ + --request "$CI_RUN_DIR/prepared-request.json" --case '${{ matrix.id }}' \ + --patch "$CI_RUN_DIR/cases/${{ matrix.id }}/prepared/materialization.patch" \ + --run-dir "$CI_RUN_DIR" + - name: Preserve formatting failure + if: ${{ matrix.enabled && always() }} + run: | + fragment="$CI_RUN_DIR/fragments/${{ matrix.id }}.treefmt.json" + if [[ ! -f "$fragment" ]]; then + python3 scripts/ci/report/fragment.py \ + --id '${{ matrix.id }}.treefmt' --label '${{ matrix.id }}: Formatting' \ + --kind validation --status failure --headline 'formatting task crashed' \ + --out "$fragment" + fi + - name: Upload formatting state + if: ${{ matrix.enabled && always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-state-treefmt-${{ matrix.id }}-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }}/fragments/${{ matrix.id }}.treefmt.json + retention-days: 30 + if-no-files-found: warn + - name: Upload formatting log + if: ${{ matrix.enabled && steps.formatting.outcome == 'failure' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-logs-treefmt-${{ matrix.id }}-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }}/cases/${{ matrix.id }}/logs/treefmt + retention-days: 30 + if-no-files-found: ignore - # consumed by test-report.yml (workflow_run, so fork PRs get the check - # run and sticky comment too); uploaded before the build so a timeout - # still leaves the eval side of the report - - name: Upload rebuild diff + eval: + name: Eval (${{ matrix.id }}) + needs: prepare + runs-on: depot-ubuntu-24.04-16 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.prepare.outputs.eval || '[{"id":"disabled","enabled":false}]') }} + steps: + - name: Checkout orchestrator + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install Nix + uses: cachix/install-nix-action@v31.9.1 + - name: Download prepared state + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ci-state-prepare-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + - name: Evaluate case + if: matrix.enabled + id: task + continue-on-error: true + run: | + nix run .#scripts.ci -- eval \ + --request "$CI_RUN_DIR/prepared-request.json" --case '${{ matrix.id }}' \ + --patch "$CI_RUN_DIR/cases/${{ matrix.id }}/prepared/materialization.patch" \ + --run-dir "$CI_RUN_DIR" + - name: Preserve task failure + if: ${{ matrix.enabled && always() }} + run: | + fragment="$CI_RUN_DIR/fragments/${{ matrix.id }}.eval.json" + if [[ ! -f "$fragment" ]]; then + python3 scripts/ci/report/fragment.py \ + --id '${{ matrix.id }}.eval' --label '${{ matrix.id }}: Evaluation' \ + --kind eval --status failure --headline 'evaluation task crashed' \ + --out "$fragment" + fi + - name: Upload eval state + if: ${{ matrix.enabled && always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: rebuild-diff + name: ci-state-eval-${{ matrix.id }}-${{ github.run_attempt }} path: | - rebuild-diff.md - diff-summary.json - - # the rebuild count predicts how long the build below will take, so put - # it on the PR now: check run in_progress + comment, updated in place by - # the final Post report. Fork PRs (read-only token) only get the final - # report, via test-report.yml. - - name: Post eval report - if: ${{ github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository }} - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - PRELIMINARY: "1" + ${{ env.CI_RUN_DIR }}/cases/${{ matrix.id }}/maps + ${{ env.CI_RUN_DIR }}/fragments/${{ matrix.id }}.eval.json + retention-days: 30 + if-no-files-found: warn + + report-eval: + name: Report validation and evaluation + needs: [prepare, treefmt, eval] + if: ${{ always() && needs.prepare.result == 'success' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Download current state + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - script: | - const post = require('${{ github.workspace }}/scripts/post-report.js'); - await post({github, context, core}); + pattern: ci-state-*-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + merge-multiple: true + - uses: ./.github/actions/update-ci-report + with: + run-dir: ${{ env.CI_RUN_DIR }} + app-id: ${{ vars.WASINIX_CI_APP_ID }} + app-private-key: ${{ secrets.WASINIX_CI_APP_PRIVATE_KEY }} - # publish the eval map so future PRs can diff against this commit - - name: Publish eval map - if: github.event_name == 'push' + core: + name: Core (${{ matrix.id }}) + needs: [prepare, eval] + if: ${{ always() && needs.prepare.result == 'success' }} + runs-on: depot-ubuntu-24.04-16 + timeout-minutes: 350 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.prepare.outputs.core || '[{"id":"disabled","enabled":false}]') }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v31.9.1 + - name: Download CI state + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ci-state-*-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + merge-multiple: true + - name: Build core + if: matrix.enabled + continue-on-error: true env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} + NIX_SIGNING_KEY: ${{ needs.prepare.outputs.trusted == 'true' && secrets.NIX_SIGNING_KEY || '' }} + AWS_ACCESS_KEY_ID: ${{ needs.prepare.outputs.trusted == 'true' && secrets.S3_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ needs.prepare.outputs.trusted == 'true' && secrets.S3_SECRET_ACCESS_KEY || '' }} AWS_DEFAULT_REGION: auto - run: nix run .#scripts.publish-eval-map + run: | + nix run .#scripts.ci -- build-task \ + --request "$CI_RUN_DIR/prepared-request.json" --case '${{ matrix.id }}' \ + --patch "$CI_RUN_DIR/cases/${{ matrix.id }}/prepared/materialization.patch" \ + --run-dir "$CI_RUN_DIR" --name core + - name: Preserve core failure + if: ${{ matrix.enabled && always() }} + run: | + fragment="$CI_RUN_DIR/fragments/${{ matrix.id }}.core.json" + if [[ ! -f "$fragment" ]]; then + python3 scripts/ci/report/fragment.py \ + --id '${{ matrix.id }}.core' --label '${{ matrix.id }}: Core' \ + --kind build --status failure --headline 'core task crashed' --out "$fragment" + fi + - name: Upload core state + if: ${{ matrix.enabled && always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-state-core-${{ matrix.id }}-${{ github.run_attempt }} + path: | + ${{ env.CI_RUN_DIR }}/cases/${{ matrix.id }}/junit/core.xml + ${{ env.CI_RUN_DIR }}/fragments/${{ matrix.id }}.core.json + retention-days: 30 + if-no-files-found: warn + - name: Upload core failure logs + if: ${{ matrix.enabled && always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-logs-core-${{ matrix.id }}-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }}/cases/${{ matrix.id }}/logs/core + retention-days: 30 + if-no-files-found: ignore - # per-package build + JUnit report; uploads to cache as it builds when - # credentials are present (forks build keyless), so timeouts don't lose work + report-core: + name: Report core + needs: [prepare, core] + if: ${{ always() && needs.prepare.result == 'success' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ci-state-*-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + merge-multiple: true + - uses: ./.github/actions/update-ci-report + with: + run-dir: ${{ env.CI_RUN_DIR }} + app-id: ${{ vars.WASINIX_CI_APP_ID }} + app-private-key: ${{ secrets.WASINIX_CI_APP_PRIVATE_KEY }} + + packages: + name: Packages (${{ matrix.id }}) + needs: [prepare, core] + if: ${{ always() && needs.prepare.result == 'success' }} + runs-on: depot-ubuntu-24.04-16 + timeout-minutes: 350 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.prepare.outputs.packages || '[{"id":"disabled","enabled":false}]') }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v31.9.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ci-state-*-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + merge-multiple: true - name: Build packages - id: build - shell: bash - run: nix run .#scripts.ci-build - # empty on fork PRs → build runs without uploading - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} - NIX_SIGNING_KEY: ${{ secrets.NIX_SIGNING_KEY }} - JOBS_FILE: eval-jobs.jsonl - - # of the rebuilt outputs, which actually changed content vs base? - # narinfo comparison mostly; only self-referential paths download - - name: Content diff - if: ${{ always() && steps.build.conclusion != 'skipped' }} + if: matrix.enabled continue-on-error: true - shell: bash + env: + NIX_SIGNING_KEY: ${{ needs.prepare.outputs.trusted == 'true' && secrets.NIX_SIGNING_KEY || '' }} + AWS_ACCESS_KEY_ID: ${{ needs.prepare.outputs.trusted == 'true' && secrets.S3_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ needs.prepare.outputs.trusted == 'true' && secrets.S3_SECRET_ACCESS_KEY || '' }} + AWS_DEFAULT_REGION: auto run: | - nix run .#scripts.content-diff -- \ - --base-map base-map.json \ - --head-map eval-map.json \ - --junit nix-fast-build-result.xml \ - --md-out content-diff.md \ - --summary-out content-summary.json - cat content-diff.md >>"$GITHUB_STEP_SUMMARY" - - # one-line status + per-failure details from the JUnit; test-report.yml - # turns this into the check run title and the sticky comment - - name: Render report - if: ${{ always() && steps.build.conclusion != 'skipped' }} - shell: bash + nix run .#scripts.ci -- build-task \ + --request "$CI_RUN_DIR/prepared-request.json" --case '${{ matrix.id }}' \ + --patch "$CI_RUN_DIR/cases/${{ matrix.id }}/prepared/materialization.patch" \ + --run-dir "$CI_RUN_DIR" --name packages + - name: Preserve package failure + if: ${{ matrix.enabled && always() }} run: | - nix run .#scripts.ci-report -- \ - --junit nix-fast-build-result.xml \ - --jobs eval-jobs.jsonl \ - --diff-summary diff-summary.json \ - --content-summary content-summary.json \ - --notes update-notes.json \ - --md-out build-report.md \ - --json-out report.json - cat build-report.md >>"$GITHUB_STEP_SUMMARY" - - - name: Upload report - if: ${{ always() && steps.build.conclusion != 'skipped' }} + fragment="$CI_RUN_DIR/fragments/${{ matrix.id }}.packages.json" + if [[ ! -f "$fragment" ]]; then + python3 scripts/ci/report/fragment.py \ + --id '${{ matrix.id }}.packages' --label '${{ matrix.id }}: Packages' \ + --kind build --status failure --headline 'package task crashed' --out "$fragment" + fi + - name: Upload package state + if: ${{ matrix.enabled && always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: build-report + name: ci-state-packages-${{ matrix.id }}-${{ github.run_attempt }} path: | - build-report.md - report.json - content-diff.md - nix-fast-build-result.xml - if-no-files-found: error - - # post the check run + sticky comment directly: bot-created PRs (the - # pin bump) never fire workflow_run, and in-job is faster anyway. - # Fork PRs have a read-only token here; test-report.yml covers them. - - name: Post report - if: ${{ always() && steps.build.conclusion != 'skipped' && - (github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository) }} - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + ${{ env.CI_RUN_DIR }}/cases/${{ matrix.id }}/junit/packages.xml + ${{ env.CI_RUN_DIR }}/fragments/${{ matrix.id }}.packages.json + retention-days: 30 + if-no-files-found: warn + - name: Upload package failure logs + if: ${{ matrix.enabled && always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-logs-packages-${{ matrix.id }}-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }}/cases/${{ matrix.id }}/logs/packages + retention-days: 30 + if-no-files-found: ignore + + python: + name: Python (${{ matrix.id }}) + needs: [prepare, core] + if: ${{ always() && needs.prepare.result == 'success' }} + runs-on: depot-ubuntu-24.04-16 + timeout-minutes: 350 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.prepare.outputs.python || '[{"id":"disabled","enabled":false}]') }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v31.9.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ci-state-*-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + merge-multiple: true + - name: Build Python + if: matrix.enabled + continue-on-error: true env: - BUILD_OUTCOME: ${{ steps.build.outcome }} + NIX_SIGNING_KEY: ${{ needs.prepare.outputs.trusted == 'true' && secrets.NIX_SIGNING_KEY || '' }} + AWS_ACCESS_KEY_ID: ${{ needs.prepare.outputs.trusted == 'true' && secrets.S3_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ needs.prepare.outputs.trusted == 'true' && secrets.S3_SECRET_ACCESS_KEY || '' }} + AWS_DEFAULT_REGION: auto + run: | + nix run .#scripts.ci -- build-task \ + --request "$CI_RUN_DIR/prepared-request.json" --case '${{ matrix.id }}' \ + --patch "$CI_RUN_DIR/cases/${{ matrix.id }}/prepared/materialization.patch" \ + --run-dir "$CI_RUN_DIR" --name python + - name: Preserve Python failure + if: ${{ matrix.enabled && always() }} + run: | + fragment="$CI_RUN_DIR/fragments/${{ matrix.id }}.python.json" + if [[ ! -f "$fragment" ]]; then + python3 scripts/ci/report/fragment.py \ + --id '${{ matrix.id }}.python' --label '${{ matrix.id }}: Python' \ + --kind build --status failure --headline 'Python task crashed' --out "$fragment" + fi + - name: Upload Python state + if: ${{ matrix.enabled && always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-state-python-${{ matrix.id }}-${{ github.run_attempt }} + path: | + ${{ env.CI_RUN_DIR }}/cases/${{ matrix.id }}/junit/python.xml + ${{ env.CI_RUN_DIR }}/fragments/${{ matrix.id }}.python.json + retention-days: 30 + if-no-files-found: warn + - name: Upload Python failure logs + if: ${{ matrix.enabled && always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-logs-python-${{ matrix.id }}-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }}/cases/${{ matrix.id }}/logs/python + retention-days: 30 + if-no-files-found: ignore + + selected-jobs: + name: Selected jobs (${{ matrix.id }}) + needs: [prepare, eval] + if: ${{ always() && needs.prepare.result == 'success' }} + runs-on: depot-ubuntu-24.04-16 + timeout-minutes: 350 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.prepare.outputs.jobs || '[{"id":"disabled","enabled":false}]') }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v31.9.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ci-state-*-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + merge-multiple: true + - name: Build selected jobs + if: matrix.enabled + continue-on-error: true + run: | + nix run .#scripts.ci -- build-task \ + --request "$CI_RUN_DIR/prepared-request.json" --case '${{ matrix.id }}' \ + --patch "$CI_RUN_DIR/cases/${{ matrix.id }}/prepared/materialization.patch" \ + --run-dir "$CI_RUN_DIR" --name jobs + - name: Preserve selected-job failure + if: ${{ matrix.enabled && always() }} + run: | + fragment="$CI_RUN_DIR/fragments/${{ matrix.id }}.jobs.json" + if [[ ! -f "$fragment" ]]; then + python3 scripts/ci/report/fragment.py \ + --id '${{ matrix.id }}.jobs' --label '${{ matrix.id }}: Selected jobs' \ + --kind build --status failure --headline 'selected-job task crashed' --out "$fragment" + fi + - name: Upload selected-job state + if: ${{ matrix.enabled && always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-state-jobs-${{ matrix.id }}-${{ github.run_attempt }} + path: | + ${{ env.CI_RUN_DIR }}/cases/${{ matrix.id }}/junit/jobs.xml + ${{ env.CI_RUN_DIR }}/fragments/${{ matrix.id }}.jobs.json + retention-days: 30 + if-no-files-found: warn + + spot: + name: Spot (${{ matrix.id }}) + needs: prepare + if: ${{ always() && needs.prepare.result == 'success' }} + runs-on: depot-ubuntu-24.04-16 + timeout-minutes: 350 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.prepare.outputs.spot || '[{"id":"disabled","enabled":false}]') }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v31.9.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - script: | - const post = require('${{ github.workspace }}/scripts/post-report.js'); - await post({github, context, core}); + name: ci-state-prepare-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + - name: Run spot build + if: matrix.enabled + continue-on-error: true + run: | + nix run .#scripts.ci -- spot-task \ + --request "$CI_RUN_DIR/prepared-request.json" --case '${{ matrix.id }}' \ + --patch "$CI_RUN_DIR/cases/${{ matrix.id }}/prepared/materialization.patch" \ + --run-dir "$CI_RUN_DIR" + - name: Preserve spot failure + if: ${{ matrix.enabled && always() }} + run: | + fragment="$CI_RUN_DIR/fragments/${{ matrix.id }}.spot.json" + if [[ ! -f "$fragment" ]]; then + python3 scripts/ci/report/fragment.py \ + --id '${{ matrix.id }}.spot' --label '${{ matrix.id }}: Spot' \ + --kind spot --status failure --headline 'spot task crashed' --out "$fragment" + fi + - name: Upload spot state + if: ${{ matrix.enabled && always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-state-spot-${{ matrix.id }}-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }}/fragments/${{ matrix.id }}.spot.json + retention-days: 30 + if-no-files-found: warn + + report-builds: + name: Report builds + needs: [prepare, packages, python, selected-jobs, spot] + if: ${{ always() && needs.prepare.result == 'success' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ci-state-*-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + merge-multiple: true + - uses: ./.github/actions/update-ci-report + with: + run-dir: ${{ env.CI_RUN_DIR }} + app-id: ${{ vars.WASINIX_CI_APP_ID }} + app-private-key: ${{ secrets.WASINIX_CI_APP_PRIVATE_KEY }} + + compare: + name: Compare (${{ matrix.id }}) + needs: [prepare, core, packages, python, selected-jobs] + if: ${{ always() && needs.prepare.result == 'success' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.prepare.outputs.compare || '[{"id":"disabled","enabled":false}]') }} + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ci-state-*-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + merge-multiple: true + - name: Compare cases + if: matrix.enabled + continue-on-error: true + run: | + python3 scripts/ci/cli.py compare \ + --request "$CI_RUN_DIR/prepared-request.json" \ + --candidate '${{ matrix.id }}' --run-dir "$CI_RUN_DIR" + - name: Preserve comparison failure + if: ${{ matrix.enabled && always() }} + run: | + fragment="$CI_RUN_DIR/fragments/compare.${{ matrix.id }}.json" + if [[ ! -f "$fragment" ]]; then + python3 scripts/ci/report/fragment.py \ + --id 'compare.${{ matrix.id }}' --label 'Compare ${{ matrix.id }}' \ + --kind comparison --status failure --headline 'comparison task crashed' \ + --out "$fragment" + fi + - name: Upload comparison state + if: ${{ matrix.enabled && always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-state-compare-${{ matrix.id }}-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }}/fragments/compare.${{ matrix.id }}.json + retention-days: 30 + + content-diff: + name: Content diff (${{ matrix.id }}) + needs: [prepare, compare] + if: ${{ always() && needs.prepare.result == 'success' }} + continue-on-error: true + runs-on: depot-ubuntu-24.04-16 + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.prepare.outputs.content || '[{"id":"disabled","enabled":false}]') }} + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v31.9.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ci-state-*-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + merge-multiple: true + - name: Compare built content + if: matrix.enabled + continue-on-error: true + run: | + nix run .#scripts.ci -- content \ + --request "$CI_RUN_DIR/prepared-request.json" \ + --candidate '${{ matrix.id }}' --run-dir "$CI_RUN_DIR" + - name: Preserve advisory failure + if: ${{ matrix.enabled && always() }} + run: | + fragment="$CI_RUN_DIR/fragments/content-diff.${{ matrix.id }}.json" + if [[ ! -f "$fragment" ]]; then + python3 scripts/ci/report/fragment.py \ + --id 'content-diff.${{ matrix.id }}' --label 'Content diff: ${{ matrix.id }}' \ + --kind analysis --status neutral --headline 'content-diff task crashed' \ + --out "$fragment" + fi + - name: Upload content-diff state + if: ${{ matrix.enabled && always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ci-state-content-${{ matrix.id }}-${{ github.run_attempt }} + path: | + ${{ env.CI_RUN_DIR }}/fragments/content-diff.${{ matrix.id }}.json + ${{ env.CI_RUN_DIR }}/comparisons/${{ matrix.id }}/content + retention-days: 14 + if-no-files-found: warn + + final-report: + name: Final report + needs: + [ + prepare, + treefmt, + eval, + core, + packages, + python, + selected-jobs, + spot, + compare, + content-diff, + ] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Download all task state + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ci-state-*-${{ github.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + merge-multiple: true + - name: Publish final report + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + uses: ./.github/actions/update-ci-report + with: + run-dir: ${{ env.CI_RUN_DIR }} + fallback-conclusion: failure + app-id: ${{ vars.WASINIX_CI_APP_ID }} + app-private-key: ${{ secrets.WASINIX_CI_APP_PRIVATE_KEY }} + - name: Enforce required conclusion + shell: bash + run: | + python3 scripts/ci/report/render.py \ + --plan "$CI_RUN_DIR/plan.json" --fragments "$CI_RUN_DIR/fragments" \ + --fallback-conclusion failure --md-out "$CI_RUN_DIR/report/report.md" \ + --json-out "$CI_RUN_DIR/report/report.json" + [[ $(jq -r .conclusion "$CI_RUN_DIR/report/report.json") == success ]] diff --git a/.github/workflows/ci-command-listener.yml b/.github/workflows/ci-command-listener.yml new file mode 100644 index 00000000..5191aae3 --- /dev/null +++ b/.github/workflows/ci-command-listener.yml @@ -0,0 +1,22 @@ +name: CI command listener + +on: + issue_comment: + types: [created] + +permissions: + contents: read + +jobs: + command: + if: >- + github.event.issue.pull_request && + startsWith(github.event.comment.body, '@wasinix ') + uses: ./.github/workflows/ci-command.yml + with: + app-id: ${{ vars.WASINIX_CI_APP_ID }} + allowed-owner: ${{ github.repository_owner }} + wasinix-repository: ${{ github.repository }} + wasinix-ref: main + secrets: + app-private-key: ${{ secrets.WASINIX_CI_APP_PRIVATE_KEY }} diff --git a/.github/workflows/ci-command.yml b/.github/workflows/ci-command.yml new file mode 100644 index 00000000..2cd17b14 --- /dev/null +++ b/.github/workflows/ci-command.yml @@ -0,0 +1,109 @@ +name: Wasinix CI command + +on: + workflow_call: + inputs: + app-id: + description: ID of the installed wasinix CI GitHub App + required: true + type: string + allowed-owner: + description: Organization allowed to submit commands + required: false + type: string + default: wasix-org + wasinix-repository: + description: Repository containing the CI orchestrator + required: false + type: string + default: wasix-org/wasinix + wasinix-ref: + description: Trusted orchestrator ref and workflow ref + required: false + type: string + default: main + secrets: + app-private-key: + description: Private key of the wasinix CI GitHub App + required: true + +permissions: + contents: read + +jobs: + dispatch: + if: >- + github.event_name == 'issue_comment' && + github.event.action == 'created' && + github.event.issue.pull_request && + startsWith(github.event.comment.body, '@wasinix ') && + github.repository_owner == inputs.allowed-owner + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Mint source token + id: source-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ inputs.app-id }} + private-key: ${{ secrets.app-private-key }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.repository }} + permission-contents: read + permission-issues: write + permission-metadata: read + permission-pull-requests: read + + - name: Mint orchestrator token + id: orchestrator-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ inputs.app-id }} + private-key: ${{ secrets.app-private-key }} + owner: ${{ inputs.allowed-owner }} + repositories: ${{ inputs.wasinix-repository }} + permission-actions: write + permission-contents: read + + - name: Checkout trusted command handler + uses: actions/checkout@v4 + with: + repository: ${{ inputs.wasinix-repository }} + ref: ${{ inputs.wasinix-ref }} + token: ${{ steps.orchestrator-token.outputs.token }} + path: .wasinix-ci + persist-credentials: false + + - name: Authorize and parse command + env: + GH_TOKEN: ${{ steps.source-token.outputs.token }} + run: | + python3 .wasinix-ci/scripts/ci/comment.py \ + --event "$GITHUB_EVENT_PATH" --allowed-owner '${{ inputs.allowed-owner }}' \ + --out "$RUNNER_TEMP/wasinix-command.json" + + - name: Dispatch wasinix CI + env: + GH_TOKEN: ${{ steps.orchestrator-token.outputs.token }} + TARGET_REPOSITORY: ${{ inputs.wasinix-repository }} + TARGET_REF: ${{ inputs.wasinix-ref }} + run: | + jq -n \ + --arg ref "$TARGET_REF" \ + --arg command "$(jq -r .command "$RUNNER_TEMP/wasinix-command.json")" \ + --arg origin "$(jq -c .origin "$RUNNER_TEMP/wasinix-command.json")" \ + --arg concurrency "$(jq -r .concurrency "$RUNNER_TEMP/wasinix-command.json")" \ + '{ref: $ref, inputs: {command: $command, origin: $origin, concurrency: $concurrency}}' \ + >"$RUNNER_TEMP/wasinix-dispatch.json" + gh api --method POST \ + "repos/$TARGET_REPOSITORY/actions/workflows/build.yml/dispatches" \ + --input "$RUNNER_TEMP/wasinix-dispatch.json" + + - name: Acknowledge command + env: + GH_TOKEN: ${{ steps.source-token.outputs.token }} + run: | + repository=$(jq -r .origin.repository "$RUNNER_TEMP/wasinix-command.json") + comment=$(jq -r .origin.commentId "$RUNNER_TEMP/wasinix-command.json") + gh api --method POST "repos/$repository/issues/comments/$comment/reactions" \ + -f content=eyes diff --git a/.github/workflows/test-report.yml b/.github/workflows/test-report.yml index 9a2c9e57..3c5444ee 100644 --- a/.github/workflows/test-report.yml +++ b/.github/workflows/test-report.yml @@ -1,11 +1,7 @@ name: CI report -# Fork-PR half of the CI report: in a pull_request run from a fork the token -# is read-only, so build.yml can't post the check run or comment itself. -# workflow_run re-executes in base-repo context with write perms. Same-repo -# events (branches, the bot's pin-bump PRs, main, merge queue) are posted -# in-job by build.yml; bot-created PRs never fire workflow_run at all (events -# from GITHUB_TOKEN don't cascade), which is why this can't be the only path. +# Fork PRs cannot update checks/comments from the pull_request workflow. Once the +# workflow completes, rerender every surviving task fragment in base-repo context. on: workflow_run: workflows: [Build] @@ -13,9 +9,9 @@ on: permissions: contents: read - actions: read # download artifacts from the triggering run - checks: write # create the check run - pull-requests: write # sticky comment + actions: read + checks: write + pull-requests: write jobs: report: @@ -23,29 +19,24 @@ jobs: if: >- github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.head_repository.full_name != github.repository + env: + CI_RUN_DIR: /tmp/wasinix-ci/${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} steps: - # for scripts/post-report.js (default-branch code, not the PR's) - name: Checkout uses: actions/checkout@v4 - # a run cancelled early may lack either artifact; post-report.js renders - # whatever is present - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - name: Download CI state + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 continue-on-error: true with: - name: rebuild-diff + pattern: ci-state-*-${{ github.event.workflow_run.run_attempt }} + path: ${{ env.CI_RUN_DIR }} + merge-multiple: true run-id: ${{ github.event.workflow_run.id }} github-token: ${{ github.token }} - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - continue-on-error: true - with: - name: build-report - run-id: ${{ github.event.workflow_run.id }} - github-token: ${{ github.token }} - - - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + - name: Post report + uses: ./.github/actions/update-ci-report with: - script: | - const post = require('${{ github.workspace }}/scripts/post-report.js'); - await post({github, context, core}); + run-dir: ${{ env.CI_RUN_DIR }} + fallback-conclusion: ${{ github.event.workflow_run.conclusion }} diff --git a/.gitignore b/.gitignore index fb7283e3..fd26090e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,5 @@ result result-* __pycache__/ *.pyc -nix-fast-build-result.xml -nix-fast-build-result.json +.ci-run/ .remote-builder diff --git a/AGENTS.md b/AGENTS.md index 9e5aba84..04a3d01e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,8 @@ scripts/update.py pin updater (nix run .#scripts.update) (`supportedProfiles`, `preferredProfile`, `broken = "reason"`; see `pkgs/lib/default.nix`). Never set `meta.badPlatforms`/`meta.broken` directly. +- Registry packagers expose `{version, rel}` as `passthru.wasix.publication`; + package files do not set it by hand. - Package placement: name in `trivial.nix` / flat `packages/.nix` / `packages//package.nix` dir; a dir's `package.nix` may be `{names, packages}` for version families (icu). Same in @@ -57,7 +59,7 @@ scripts/update.py pin updater (nix run .#scripts.update) ## Remote builds -The toolchain and the full `.ci` sweep are expensive; building them locally is +The toolchain and the full `.ciSets.all` sweep are expensive; building them locally is painful, and a stray system-default builder (e.g. a paid `nixbuild.net`) can cost real money. Route expensive builds to a remote builder you control. @@ -71,8 +73,8 @@ when you do this. - `scripts/remote-builder.sh check`: configured and reachable? - Bulk: `nix-fast-build --skip-cached --flake -.#legacyPackages.x86_64-linux.ci --store "$(scripts/remote-builder.sh -store)"` (local eval, remote build), or `scripts/ci-build-remote.sh` for the +.#legacyPackages.x86_64-linux.ciSets.all --store "$(scripts/remote-builder.sh +store)"` (local eval, remote build), or `scripts/ci/tasks/build-remote.sh` for the signed, cache-pushing CI set. - Single build: `nix build --max-jobs 0 --builders "$(scripts/remote-builder.sh builders)" --builders-use-substitutes`. @@ -87,8 +89,9 @@ with `ssh "$(scripts/remote-builder.sh host)"` + `nix log`, not a rebuild. - `git add` new files before `nix build`/`nix eval`; the flake only sees tracked files. - `nix fmt` before committing; CI rejects unformatted files. -- Job list: `nix eval .#legacyPackages.x86_64-linux.ci --apply -builtins.attrNames`. For behaviour-preserving refactors, also diff +- Job list: `nix eval .#legacyPackages.x86_64-linux.ciSets.all --apply +builtins.attrNames`; replace `all` with `core`, `packages`, or `python` for one + scheduling unit. For behaviour-preserving refactors, also diff `--apply 'j: builtins.mapAttrs (_: d: d.drvPath) j'` before/after; meta and passthru changes don't move drv paths. - A CI job name is a build path: `nix build .#librariesByProfile.exnrefEh.zlib`, @@ -102,9 +105,9 @@ builtins.attrNames`. For behaviour-preserving refactors, also diff base revision (`docs/spot.md`). Experiments only: it mixes two toolchains, so confirm at the root before keeping the change. - The most thorough check is `nix-fast-build --flake -.#legacyPackages.x86_64-linux.ci --no-link --skip-cached --option +.#legacyPackages.x86_64-linux.ciSets.all --no-link --skip-cached --option accept-flake-config true`, the same build set as CI - (`scripts/ci-build.sh`). `--skip-cached` only helps while the change + (`scripts/ci/tasks/build.sh`). `--skip-cached` only helps while the change avoids mass rebuilds, and those are easy to trigger (anything under `pkgs/toolchain/`, a pin bump); expect a huge build that can OOM the machine. Do not run it without asking the user first. diff --git a/README.md b/README.md index 0d3cca0b..8e6e164d 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,12 @@ nix build .#legacyPackages.x86_64-linux.allWasmerPackages # al nix run .#scripts.update # bump the source pins ``` -CI builds every package as its own job (`.#legacyPackages..ci`, -driven by `scripts/ci-build.sh`). A job's dotted name is its build path. +CI roots are partitioned under `.#legacyPackages..ciSets`: `core` +validates the toolchain, `packages` covers C/C++/Rust libraries and programs, +and `python` covers wheels and the registry. `ciSets.all` is their disjoint +union. A job's dotted name is its build path. Use the same CI command locally +and in automation, for example `nix run .#scripts.ci -- build core` or +`nix run .#scripts.ci -- diff build core --at main --vs build core --at HEAD`. ## Structure @@ -43,6 +47,7 @@ Details: [`docs/architecture.md`](docs/architecture.md). | ---------------------------------------------- | -------------------------------------------- | | [`AGENTS.md`](AGENTS.md) | conventions and rules for making changes | | [`docs/architecture.md`](docs/architecture.md) | how the layers fit together | +| [`docs/ci.md`](docs/ci.md) | CI sets, task state, reports, and retention | | [`docs/packaging.md`](docs/packaging.md) | adding packages: C, CLI/webc, Rust, Python | | [`docs/updating.md`](docs/updating.md) | the pin updater | | [`docs/spot.md`](docs/spot.md) | experimenting without rebuilding the world | diff --git a/docs/architecture.md b/docs/architecture.md index 0bf8d026..57948ad6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -195,19 +195,33 @@ patch tree, so the two can't drift: `wasmerPackages.` (with `.pkg`, `.webc`, `.tests`), `pythonWheels..`; plus `nixpkgsByProfile`, `toolchainByProfile`, `pkgsCross`, `allWasmerPackages`. -- `ci`: the same trees flattened to dotted names, so a job name is a build - path. Unsupported/broken packages are filtered out before becoming jobs. - `scripts/ci-build.sh` runs it with nix-fast-build and incremental cache - upload. `scripts/eval-diff.py` diffs the eval (attr to drvPath) against the - base branch to surface what a PR rebuilds; maps are published to the cache - bucket (`eval-maps/.json`) on pushes to main. `scripts/content-diff.py` - then splits rebuilt outputs into bit-identical vs actually changed +- `ciSets`: the buildable trees flattened to dotted job names and partitioned + into disjoint scheduling units: `core` (toolchain), `packages` + (C/C++/Rust libraries, CLIs, webcs, ABI validation and tests), and `python` + (wheels, import tests and the registry). `all` is their explicit union. + Unsupported/broken packages are filtered out before becoming jobs. + `scripts/ci/tasks/build.sh` takes a set name (or `CI_SET`) and runs it with + nix-fast-build and incremental cache upload. Formatting is a standalone, + candidate-side validation: required in the final report but parallel with + eval and builds. Content diff is an advisory job enabled explicitly. + `scripts/ci/tasks/eval-diff.py` diffs `ciSets.all` (attr to drvPath) to + surface what moved; maps include each job's set membership, version, + publication rel, and CI policy. Artifact outputs are content-comparable; + `checks.*` are validation jobs whose aggregate outputs are only success + tokens. `scripts/ci/tasks/content-diff.py` then splits rebuilt artifact + outputs into bit-identical vs actually changed (narinfo narHash compare; self-referential paths get normalized with `nix -store make-content-addressed`). `scripts/ci-report.py` folds all that and - the JUnit results into the "Per-package status" check run and a sticky PR - comment (scripts/post-report.js): posted in-job for same-repo events (the +store make-content-addressed`). Tasks emit schema-versioned fragments under a + per-run state directory; `scripts/ci/report/render.py` folds every available + fragment into the "Per-package status" check run and a sticky PR comment via + `scripts/ci/report/post.js`: posted in-job for same-repo events (the bot's pin-bump PRs never fire workflow_run), via test-report.yml for fork - PRs, where the in-job token is read-only. + PRs, where the in-job token is read-only. See `docs/ci.md`. +- **CI commands** (`scripts/ci/comment.py`, `scripts/ci/origin.py`): the shared + issue-comment workflow authenticates a write-level caller, binds the source + PR to an immutable head, and dispatches the ordinary CI command language. + The App credential exists only in listener and report jobs. Originated build + jobs cannot sign or publish cache entries. CA derivations were considered (early cutoff would show which rebuilds actually change outputs) and rejected for now: binary caches cannot serve diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 00000000..7179039b --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,218 @@ +# CI commands, tasks, and state + +The local command and the GitHub workflow use the same entry point: + +```sh +nix run .#scripts.ci -- build core +nix run .#scripts.ci -- build core packages +nix run .#scripts.ci -- build python --with wasix-libc=rev:<40-character-sha> +nix run .#scripts.ci -- build attr:wasmerPackages.git.webc +``` + +The disjoint build roots are `legacyPackages..ciSets.{core,packages,python}`. +`core` validates the toolchain, `packages` covers C/C++/Rust libraries and +programs, and `python` covers wheels and the registry. `all` expands to all +three. Requesting packages or Python also schedules core first. Exact +`attr:` selectors do not implicitly expand a set. + +Builds use the configured `.remote-builder` by default. Pass `--local` only +when the selected work is cheap or the command is already running on a +dedicated builder. + +## Sources and overrides + +Every source is normalized to a commit SHA before work begins. `--at REF` +selects the wasinix source. An override is materialized by the normal updater +in a temporary worktree, so hashes and generated pins change exactly as they +would in a committed update: + +```sh +nix run .#scripts.ci -- build python \ + --at main \ + --with wasix-libc=rev:<40-character-sha> + +nix run .#scripts.ci -- build packages \ + --with wasmer=version:7.1.0 +``` + +`--from-pr=OWNER/REPO#NUMBER` is syntax sugar. A wasinix PR selects that PR's +source commit. A PR in a repository named by a package's +`passthru.updateScript.source` becomes a revision override for that package. +Bare `--from-pr` reads the current GitHub pull-request event. Ambiguous source +mappings are rejected. + +Materialization captures tracked working-tree changes when `--at HEAD` is +used. Untracked files are rejected because flakes cannot reproduce them. The +prepared patch, its hash, and the immutable source SHA are saved with the run. + +## Spot requests + +Spot remains an experiment that splices a changed target over a cached base, +but it uses the same source and override materialization: + +```sh +nix run .#scripts.ci -- spot attr:exnrefEh.zlib \ + --base --keep toolchain,zlib +``` + +The target syntax is deliberately explicit. A green spot result is evidence, +not a shipping verdict; confirm the change at the normal root before keeping +it. See [spot.md](spot.md). + +## Differential builds + +`diff` compares two or more complete build commands. The first command is the +baseline and every later command is independently compared with it: + +```sh +nix run .#scripts.ci -- diff \ + build python --at main \ + --vs \ + build python --at HEAD --with wasix-libc=rev:<40-character-sha> +``` + +Each case is evaluated and built in an isolated worktree. Results align by +stable CI job name, not by matrix position. The comparison reports: + +- new build failures and new evaluation errors as regressions; +- fixed and pre-existing failures separately; +- rebuilt derivations and changed version/publication-rel identities; +- jobs added to or removed from the selected request. + +Known failures reproduced by both cases do not fail the comparison. Missing +case results do. Content comparison is a separate advisory task because it can +be slow and requires both output closures: + +```sh +nix run .#scripts.ci -- diff --content-diff \ + build packages --at main --vs build packages --at HEAD +``` + +## GitHub execution + +Every CI job carries policy in the eval map. Artifact jobs participate in build +status, rebuild, and content comparison. Validation jobs participate in build +status and rebuild comparison, but their success-token outputs are excluded +from content comparison. This is declared by `role`, `rebuildSignal`, and +`contentDiff` in `ciJobInfo`; consumers do not infer meaning from output size or +NAR contents. + +The `Build` workflow is a thin phase runner around `.#scripts.ci`: + +```text + +-> formatting ------------------------------+ +prepare --------+-> eval -> core -> packages / python / jobs -> compare + -> content diff +``` + +Prepare normalizes the request, materializes every case, writes the task plan, +and exports per-task matrices. Each later job consumes the prepared request and +patch, then emits a versioned JSON fragment. Formatting runs on submitted or +candidate materializations, never the trusted diff baseline. It is required for +the final verdict but does not gate eval or builds, so expensive work can warm +the cache while formatting is repaired. CI builds the submitted materialization +unchanged; it never silently formats a hypothetical checkout whose hashes may +differ from the reviewed commit. Core gates the semantic package and Python +sets; exact jobs only depend on eval. Content diff is opt-in and advisory. The +unified renderer runs after formatting, evaluation, core, the remaining builds, +and final analysis, reading every fragment available at that point. + +Pull requests automatically compare `core packages` against their base. Merge +groups compare `all`; pushes to main build `all`; `workflow_dispatch` accepts +the same command string as the local CLI. The `ci:content-diff` PR label opts a +pull request into content comparison. Synchronizing a PR, including an update +PR, starts a fresh build automatically. + +External automation can send a `repository_dispatch` event of type +`wasinix-ci-v1` whose `client_payload.request` is a normalized request object: + +```json +{ + "event_type": "wasinix-ci-v1", + "client_payload": { + "head_sha": "", + "concurrency": "optional-caller-key", + "request": { + "schema": 1, + "action": "build", + "source": { + "rev": "<40-character-sha>", + "patch": null, + "workingTree": false + }, + "selectors": [{ "kind": "set", "name": "python" }], + "overrides": [ + { + "target": "wasix-libc", + "kind": "revision", + "value": "<40-character-sha>" + } + ], + "execution": { "local": false } + } + } +} +``` + +The dispatch boundary rejects moving refs, caller-provided patches, local/dry +execution, duplicate targets, and diffs larger than four cases. Dispatched +builds never receive signing or publication secrets. + +The versioned schemas live in `scripts/ci/report/schemas/`. + +## Pull-request comment commands + +Selected `wasix-org` repositories can call the same command language through a +GitHub App: + +```text +@wasinix build python --from-pr +@wasinix diff build packages --at main --vs build packages --from-pr +``` + +The shared listener accepts commands only from users with write permission, +binds bare `--from-pr` to the originating PR's immutable head SHA, and starts +this workflow with a validated origin record. Reports are written back as a +check and bot comment on the originating PR. These runs cannot sign or publish +cache entries. See [github-app.md](github-app.md) for App permissions and the +caller workflow. + +## Run directory + +Generated state lives under `CI_RUN_DIR`, `/tmp/wasinix-ci/-` in +GitHub and `.ci-run` by default locally: + +```text +request.json +origin.json # only for PR-comment commands +prepared-request.json +plan.json +matrix.json +cases// + prepared/{request.json,materialization.json,materialization.patch} + maps/{eval-jobs.jsonl,eval-map.json,...} + junit/.xml + logs//{manifest.json,.log.gz} + logs/treefmt/treefmt.log +comparisons//content/{summary.json,content-diff.md} +fragments/.json +report/{report.json,report.md} +``` + +`plan.json` declares enabled tasks and whether they are blocking. Task +fragments contain only the common status envelope plus task-owned data and +Markdown. The renderer is a pure aggregation step. Eval maps carry job +versions and, for published wheels and webcs, publication rels. + +## Retention + +- Prepared requests, plans, fragments, JUnit, and eval maps are retained for + 30 days. +- Bounded failure logs are retained for 30 days. Each gzip log is capped at + 20 MiB and each task at 100 MiB. +- Advisory content-diff state is retained for 14 days. + +Successful builds do not create log artifacts. `manifest.json` maps archived +logs back to attrs and derivations and records truncation. The binary cache has +its own reachability policy; CI artifact age must never be used to collect NARs +or narinfos. diff --git a/docs/github-app.md b/docs/github-app.md new file mode 100644 index 00000000..03fd5a6e --- /dev/null +++ b/docs/github-app.md @@ -0,0 +1,95 @@ +# GitHub App CI commands + +The wasinix CI command language can be called from pull-request comments in +selected `wasix-org` repositories: + +```text +@wasinix build python --from-pr +@wasinix build attr:wasmerPackages.git.webc --from-pr +@wasinix build python --with wasix-libc=rev:<40-character-sha> +@wasinix diff build packages --at main --vs build packages --from-pr +``` + +Commands are one physical line and use the same parser as `nix run +.#scripts.ci`. Bare `--from-pr` means the pull request containing the comment. +The listener records that PR's current head SHA before dispatch, so later +synchronization cannot change what the accepted command builds. + +## App registration + +Create a GitHub App owned by `wasix-org`, grant these repository permissions, +and install it only on `wasix-org/wasinix` and repositories allowed to submit +commands: + +- Actions: write +- Checks: write +- Contents: read +- Issues: write +- Pull requests: read +- Metadata: read, granted implicitly by GitHub + +Actions write starts `build.yml` in `wasix-org/wasinix`. Issues and Checks +write publish the result back on the originating pull request. Contents and +Pull requests read resolve the trusted handler and immutable PR head. The +workflows mint installation tokens scoped to only the repositories and +permissions used by that step. + +Store the App ID in the organization Actions variable +`WASINIX_CI_APP_ID`. Store a private key in the organization Actions secret +`WASINIX_CI_APP_PRIVATE_KEY`. Limit both to the selected repositories. The +private key is used only by the command listener and report publisher; build +jobs never receive it. + +The wasinix repository already contains `ci-command-listener.yml`. Add this +small caller workflow to each other repository: + +```yaml +name: Wasinix CI commands + +on: + issue_comment: + types: [created] + +permissions: + contents: read + +jobs: + command: + if: >- + github.event.issue.pull_request && + startsWith(github.event.comment.body, '@wasinix ') + uses: wasix-org/wasinix/.github/workflows/ci-command.yml@main + with: + app-id: ${{ vars.WASINIX_CI_APP_ID }} + secrets: + app-private-key: ${{ secrets.WASINIX_CI_APP_PRIVATE_KEY }} +``` + +Pin `@main` to a wasinix commit if the caller repository requires immutable +reusable-workflow references. Updating that pin is the only per-repository +maintenance needed. + +## Authorization and reporting + +The reusable workflow runs from trusted default-branch code. It accepts only +new pull-request comments beginning exactly with `@wasinix ` and asks GitHub +for the commenter's effective repository permission. `write`, `maintain`, and +`admin` are accepted; `read` and `triage` are rejected. The command parser also +rejects `--local` and spot `--dry-run` execution. + +An accepted command receives an eyes reaction. The App starts the trusted +`build.yml` from the configured wasinix ref with: + +- the original command string; +- the source repository, PR, comment, actor, and immutable head SHA; +- a per-PR concurrency key, so a newer command supersedes an older run. + +The target workflow validates that origin again and only permits repositories +under its own organization. Originated runs do not receive Nix signing or +cache-publication credentials. Each report pass mints a separate token scoped +to the source repository, verifies that the PR still has the recorded head, +then updates a `Wasinix CI` check and a bot comment tied to the command comment. + +The origin format is versioned by +`scripts/ci/report/schemas/command-origin-v1.json`. Caller workflows should not +construct normalized build requests or duplicate command parsing. diff --git a/docs/updating.md b/docs/updating.md index 42920345..5f14021a 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -4,8 +4,26 @@ nix run .#scripts.update # everything nix run .#scripts.update -- --list # targets + current pins nix run .#scripts.update -- --only llvm wasix-libc +nix run .#scripts.update -- --to wasix-libc=2026-08-01.1 +nix run .#scripts.update -- --to-rev wasix-libc= ``` +`--to NAME=VERSION` asks a target to materialize an exact upstream release. +`--to-rev NAME=SOURCE` instead materializes an immutable source revision +for CI without changing its release identity. A revision source is either a +40-character commit SHA, using the repository declared by the target, or +`github:OWNER/REPO@SHA`. Symbolic names such as `PR_HEAD` must be resolved to a +commit before invoking the updater. + +Explicit updates are opt-in. The package declares the accepted request modes +and source repository beside its `passthru.updateScript`; the driver rejects +unsupported targets and repositories before running anything. Revision runs +skip registry-history retention, `rels.json` pruning, retention hooks, and +update notes. They leave an ordinary working-tree change containing the source +revision, source hash, and any package-owned derived pins. `wasix-libc` is the +first revision-capable target; its updater also derives both witx submodule pins +from the requested revision. + How a pin bumps is declared next to it (`passthru.updateScript`, the standard nixpkgs convention); its constraints and quirks are comments in the same file. That includes a pin _derived_ from another pin: the rust fork's stage0 diff --git a/flake.nix b/flake.nix index 8a7ac248..6a6b3497 100644 --- a/flake.nix +++ b/flake.nix @@ -102,17 +102,35 @@ ) ) {}; collectTests = collectTestsPrefixed ""; - flakeChecks = - collectTests wasix.wasmerPackages - // collectTests wasix.toolchainTestPkgs - # pythonWheels is nested by version (py313/py314); collect as wheel-py314-. - // lib.concatMapAttrs (pv: wheelSet: collectTestsPrefixed "wheel-${pv}-" wheelSet) wasix.pythonWheels - // collectTests {python-registry = wasix.pythonRegistry;} - // collectTests {cargo-registry = wasix.cargoRegistry;} - // lib.mapAttrs' (p: lib.nameValuePair "abi-${p}") wasix.abiChecks - # non-shipped library packages carrying a tests/ dir - // collectTests wasix.libraryTestPkgs - // {treefmt = treefmtEval.config.build.check self;}; + mergeDisjoint = context: sets: let + names = lib.concatMap builtins.attrNames sets; + duplicates = + lib.attrNames + (lib.filterAttrs (_: occurrences: lib.length occurrences > 1) + (lib.groupBy (name: name) names)); + in + lib.throwIf (duplicates != []) + "${context}: duplicate jobs (${lib.concatStringsSep ", " duplicates})" + (lib.foldl' (acc: set: acc // set) {} sets); + treefmtCheck = treefmtEval.config.build.check self; + checksBySet = { + core = mergeDisjoint "checksBySet.core" [ + (collectTests wasix.toolchainTestPkgs) + ]; + packages = mergeDisjoint "checksBySet.packages" [ + (collectTests wasix.wasmerPackages) + (collectTests {cargo-registry = wasix.cargoRegistry;}) + (lib.mapAttrs' (p: lib.nameValuePair "abi-${p}") wasix.abiChecks) + # non-shipped library packages carrying a tests/ dir + (collectTests wasix.libraryTestPkgs) + ]; + python = mergeDisjoint "checksBySet.python" [ + # pythonWheels is nested by version; collect as wheel-py314-. + (lib.concatMapAttrs (pv: wheelSet: collectTestsPrefixed "wheel-${pv}-" wheelSet) wasix.pythonWheels) + (collectTests {python-registry = wasix.pythonRegistry;}) + ]; + }; + flakeChecks = mergeDisjoint "checks" ([{treefmt = treefmtCheck;}] ++ builtins.attrValues checksBySet); in { formatter.${system} = treefmtEval.config.build.wrapper; @@ -120,7 +138,7 @@ # make `nix flake check` warn. legacyPackages.${system} = let # These attr paths are both the `.#` build targets and, flattened to dotted - # keys, the `ci` job names, so the two cannot drift. + # keys, the CI job names, so the two cannot drift. buildable = { # Profile-independent tools. Sysroot libraries and their Fortran/OpenMP # drivers live under `.#toolchainByProfile.`. @@ -183,8 +201,46 @@ then flattenDrvs key val else {} ); - # One derivation per dotted key for nix-eval-jobs / nix-fast-build. - ci = flattenDrvs "" buildable // flattenDrvs "checks" flakeChecks; + # Disjoint scheduling units. Keep these semantic: core validates the + # toolchain, packages covers C/C++/Rust programs and libraries, and python + # owns the wheel/registry matrix. `all` is the explicit full sweep. + ciSetParts = { + core = [ + (flattenDrvs "toolchain" buildable.toolchain) + (flattenDrvs "checks" checksBySet.core) + ]; + packages = [ + (flattenDrvs "librariesByProfile" buildable.librariesByProfile) + (flattenDrvs "wasmerPackages" buildable.wasmerPackages) + (flattenDrvs "" {inherit (buildable) cargoRegistry;}) + (flattenDrvs "checks" checksBySet.packages) + ]; + python = [ + (flattenDrvs "pythonWheels" buildable.pythonWheels) + (flattenDrvs "" {inherit (buildable) pythonRegistry;}) + (flattenDrvs "checks" checksBySet.python) + ]; + }; + ciSetsDisjoint = + lib.mapAttrs + (name: mergeDisjoint "ciSets.${name}") + ciSetParts; + ciSets = + ciSetsDisjoint + // {all = mergeDisjoint "ciSets.all" (builtins.attrValues ciSetsDisjoint);}; + ciJobInfo = lib.mapAttrs (name: drv: let + isCheck = lib.hasPrefix "checks." name; + in + wasixLib.ciInfoOf drv + // { + role = + if isCheck + then "check" + else "artifact"; + rebuildSignal = true; + contentDiff = !isCheck; + }) + ciSets.all; in buildable // { @@ -200,7 +256,7 @@ pkgsCross.wasix = wasix.pkgsCross; allWasmerPackages = wasix.allWasmerPackages; - inherit ci; + inherit ciSets ciJobInfo; # CI shell steps as runnable apps with nix-pinned deps: `nix run # .#scripts.`. The dir is store-copied, so no git checkout is needed @@ -235,11 +291,12 @@ ''; }; in { - ci-build = run "ci-build" [p.jq p.nix-eval-jobs p.nix-fast-build p.findutils] "bash" "ci-build.sh"; - rebuild-diff = run "rebuild-diff" [p.python3 p.nix-eval-jobs] "bash" "rebuild-diff.sh"; - content-diff = run "content-diff" [] "python3" "content-diff.py"; - ci-report = run "ci-report" [] "python3" "ci-report.py"; - publish-eval-map = run "publish-eval-map" [p.awscli2] "bash" "publish-eval-map.sh"; + ci = run "ci" [p.jq p.nix-eval-jobs p.nix-fast-build p.findutils wasix.nixUpdate p.nix-prefetch-git p.cargo] "python3" "ci/cli.py"; + ci-build = run "ci-build" [p.jq p.nix-eval-jobs p.nix-fast-build p.findutils] "bash" "ci/tasks/build.sh"; + rebuild-diff = run "rebuild-diff" [p.python3 p.nix-eval-jobs] "bash" "ci/tasks/rebuild-diff.sh"; + content-diff = run "content-diff" [] "python3" "ci/tasks/content-diff.py"; + ci-report = run "ci-report" [] "python3" "ci/report/build-fragment.py"; + publish-eval-map = run "publish-eval-map" [p.awscli2] "bash" "ci/tasks/publish-eval-map.sh"; bump-rel = run "bump-rel" [] "python3" "bump-rel.py"; publish-index = run "publish-index" [wasmerRuntime p.rclone p.python3 p.gawk p.gnused] "bash" "publish-index.sh"; publish-webc = run "publish-webc" [wasmerRuntime] "python3" "publish-webc.py"; @@ -274,7 +331,7 @@ # `versions` is published in the eval maps; `fired` gets the base branch's # copy back as the `prior` side of each note's predicate. updateNotes = let - noted = lib.filterAttrs (_: wasixLib.hasUpdateNotes) ci; + noted = lib.filterAttrs (_: wasixLib.hasUpdateNotes) ciSets.all; versionOf = drv: let r = builtins.tryEval (wasixLib.noteVersionOf drv); in @@ -340,6 +397,8 @@ {inherit command commandDrvPaths;} // lib.optionalAttrs (lib.isAttrs s && s ? name) {inherit (s) name;} // lib.optionalAttrs (lib.isAttrs s && s ? attrPath) {inherit (s) attrPath;} + // lib.optionalAttrs (lib.isAttrs s && s ? accepts) {inherit (s) accepts;} + // lib.optionalAttrs (lib.isAttrs s && s ? source) {inherit (s) source;} // {position = drv.meta.position or null;}; }; in @@ -350,7 +409,7 @@ then entry.value else {}; in - lib.concatMapAttrs scriptOf ci; + lib.concatMapAttrs scriptOf ciSets.all; # passthru.wasix.retentionHook: a command scripts/update.py runs after the # repo-wide history/prune steps. In-tree only; the driver dedupes repeats. @@ -380,7 +439,7 @@ then entry.value else {}; in - lib.concatMapAttrs hookOf ci; + lib.concatMapAttrs hookOf ciSets.all; }; devShells.${system}.default = wasix.pkgs.mkShell { diff --git a/pkgs/cargo-registry/default.nix b/pkgs/cargo-registry/default.nix index f01d5ad4..0f897cbd 100644 --- a/pkgs/cargo-registry/default.nix +++ b/pkgs/cargo-registry/default.nix @@ -88,7 +88,10 @@ passthru = { inherit crate version wasixVersion crateFile rel; - wasix.supportedProfiles = []; + wasix = { + supportedProfiles = []; + publication = {inherit version rel;}; + }; }; meta = { diff --git a/pkgs/default.nix b/pkgs/default.nix index c0d0e712..8f8d11db 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -247,13 +247,31 @@ # Shipped Python wheels (overlay/python-packages/wheels.nix). cpython needs PIC # (ctypes/dl) and the exnref EH encoding wasmer accepts, so the wheels are one set # anchored at exnrefEhpic. noarch builds once, everything else per interpreter. - mkPythonWheels = pyKey: pyAttr: webcName: select: - import ./python-wheels.nix { + publicationRels = builtins.fromJSON (builtins.readFile ../rels.json); + mkPythonWheels = pyKey: pyAttr: webcName: select: let + wheels = import ./python-wheels.nix { inherit pkgs lib mkTestGroup select pyKey; python3 = nixpkgsByProfile.exnrefEhpic.${pyAttr}; wasmer = wasmerRuntime; pythonWebc = wasmerLayer.wasmerPackages.${webcName}.webc; }; + withPublication = _: drv: + drv.overrideAttrs (o: { + passthru = + (o.passthru or {}) + // { + wasix = + ((o.passthru or {}).wasix or {}) + // { + publication = { + inherit (drv) version; + rel = (publicationRels."pythonRegistry.wheels.${drv.pname or drv.name}" or {}).${drv.version} or 1; + }; + }; + }; + }); + in + lib.mapAttrs withPublication wheels; isNoarch = e: e.noarch or false; publishOnceWheelNames = map (e: e.attr) diff --git a/pkgs/lib/default.nix b/pkgs/lib/default.nix index 20622cb3..7f8ac7df 100644 --- a/pkgs/lib/default.nix +++ b/pkgs/lib/default.nix @@ -53,6 +53,21 @@ in rec { wasixMetaOf = drv: (drv.passthru or {}).wasix or {}; + # Stable identity for human-facing CI diffs. Published artifacts override the + # derivation's upstream version with their registry version and release. + ciInfoOf = drv: let + publication = (wasixMetaOf drv).publication or {}; + version = publication.version or drv.version or null; + rel = publication.rel or null; + info = + lib.optionalAttrs (builtins.isString version) {inherit version;} + // lib.optionalAttrs (builtins.isInt rel) {inherit rel;}; + forced = builtins.tryEval (builtins.deepSeq info info); + in + if forced.success + then forced.value + else {}; + # meta.position ("file:line") as a mkDerivation `pos` argument, so generated # drvs inherit their subject's position and `nix edit` lands somewhere useful. posOf = drv: let diff --git a/pkgs/toolchain/sysroot/libc.nix b/pkgs/toolchain/sysroot/libc.nix index c403dfa0..92367b5f 100644 --- a/pkgs/toolchain/sysroot/libc.nix +++ b/pkgs/toolchain/sysroot/libc.nix @@ -86,6 +86,12 @@ in name = "wasix-libc"; # the attr tail is `libc` # Wraps nix-update (passed through as argv) to re-derive the witx pins at the new tag. command = ["pkgs/toolchain/sysroot/update.py"] ++ nix-update-script {extraArgs = ["--flake"];}; + accepts = ["release" "revision"]; + source = { + kind = "github"; + owner = "wasix-org"; + repo = "wasix-libc"; + }; }; nativeBuildInputs = [ diff --git a/pkgs/toolchain/sysroot/update.py b/pkgs/toolchain/sysroot/update.py index 5d71453c..4c3f03e4 100755 --- a/pkgs/toolchain/sysroot/update.py +++ b/pkgs/toolchain/sysroot/update.py @@ -4,11 +4,13 @@ # libc.nix pins the wasi/wasix witx specs (git submodules of wasix-libc) # separately from the libc src, because a submodule is not part of the source # tarball. A stale pin fails the build with undeclared __wasi_* functions, and -# the correct rev is whatever the new tag points its submodule at, so this is -# package knowledge and lives next to the pin it edits. +# the correct rev is whatever the selected source revision points its submodule +# at, so this is package knowledge and lives next to the pin it edits. # # Invoked as `update.py `: the driver passes the command -# nix-update-script produced, so the package declares its bump once. +# nix-update-script produced. An explicit revision request materializes the +# requested revision directly, then both paths derive the witx pins from the +# resulting source revision. import re import subprocess @@ -29,21 +31,67 @@ / "scripts" ), ) -from updater_lib import REPO, gh, prefetch_github, run_nix_update # noqa: E402 +from updater_lib import ( # noqa: E402 + REPO, + gh, + prefetch_github, + run_nix_update, + update_request, +) LIBC = REPO / "pkgs/toolchain/sysroot/libc.nix" SUBMODULES = [ ("tools/wasi-headers/WASI", "WebAssembly", "WASI"), ("tools/wasix-headers/WASI", "wasix-org", "wasix-witx"), ] +SOURCE_PIN = re.compile( + r'(repo = "wasix-libc";\s*\n\s*)(?:tag|rev)( = ")([^"]+)' + r'(";\s*\n\s*hash = ")(sha256-[^"]+)(";)', + re.S, +) + + +def source_revision(text): + match = SOURCE_PIN.search(text) + if not match: + raise SystemExit("wasix-libc source pin block not found in libc.nix") + rev = match.group(3) + if rev == "v${version}": + version = re.search(r'\bversion = "([^"]+)"', text).group(1) + return f"v{version}" + return rev + + +def materialize_revision(request): + source = request.source or {} + if source.get("kind") != "github": + raise SystemExit("wasix-libc revision update requires a GitHub source") + if (source.get("owner"), source.get("repo")) != ("wasix-org", "wasix-libc"): + raise SystemExit("wasix-libc revision source must be wasix-org/wasix-libc") + rev = source.get("rev", "") + if not re.fullmatch(r"[0-9a-f]{40}", rev): + raise SystemExit("wasix-libc revision must be a 40-character commit SHA") + + text = LIBC.read_text() + match = SOURCE_PIN.search(text) + if not match: + raise SystemExit("wasix-libc source pin block not found in libc.nix") + prior = source_revision(text) + source_hash = prefetch_github("wasix-org", "wasix-libc", rev) + replacement = ( + f"{match.group(1)}rev{match.group(2)}{rev}" + f"{match.group(4)}{source_hash}{match.group(6)}" + ) + LIBC.write_text(text[: match.start()] + replacement + text[match.end() :]) + return prior, rev -def sync_witx(): +def sync_witx(rev=None): text = LIBC.read_text() - tag = "v" + re.search(r'\bversion = "([^"]+)"', text).group(1) + rev = rev or source_revision(text) bumped = [] for sub, owner, repo in SUBMODULES: - sha = gh(f"wasix-org/wasix-libc/contents/{sub}?ref={tag}")["sha"] + sha = gh(f"wasix-org/wasix-libc/contents/{sub}?ref={rev}")["sha"] m = re.search( rf'repo = "{repo}";\s*\n\s*rev = "([^"]+)";\s*\n\s*hash = "([^"]+)"', text ) @@ -61,8 +109,14 @@ def sync_witx(): def main(): - run_nix_update(sys.argv[1:]) - synced = sync_witx() + request = update_request("wasix-libc") + if request is not None and request.mode == "revision": + prior, rev = materialize_revision(request) + print(f"{prior} -> {rev}") + else: + run_nix_update(sys.argv[1:], request) + rev = None + synced = sync_witx(rev) # No " -> ": the driver scans stdout backwards for an outcome line and must # land on nix-update's, not this one. print(f"witx pins synced: {synced}" if synced else "witx pins ok") diff --git a/pkgs/wasmer/default.nix b/pkgs/wasmer/default.nix index 3f9d179f..2f00cb04 100644 --- a/pkgs/wasmer/default.nix +++ b/pkgs/wasmer/default.nix @@ -80,6 +80,9 @@ webc = pkg.webc; # run-by-name wrapper; forcing it never forces .tests shim = pkg.webc.shim; + wasix = + ((o.passthru or {}).wasix or {}) + // {inherit (pkg.passthru.wasix) publication;}; } // (lib.optionalAttrs (group != null) {tests = group;}); }); diff --git a/pkgs/wasmer/make-wasmer-package.nix b/pkgs/wasmer/make-wasmer-package.nix index df31984b..b2d1404e 100644 --- a/pkgs/wasmer/make-wasmer-package.nix +++ b/pkgs/wasmer/make-wasmer-package.nix @@ -36,6 +36,10 @@ inherit (ident) rels webcIdent; inherit (webcIdent package) name owner version baseVersion rel; + publication = { + version = baseVersion; + inherit rel; + }; # rels.json keys no served version carries: left behind by an upstream bump; # scripts/update.py drops them (regen hook on nixpkgs), this note covers @@ -190,15 +194,19 @@ in passAsFile = ["readme"]; passthru = { id = {inherit owner name version baseVersion;}; + wasix = {inherit publication;}; inherit depWebcs; # The built webc at owner/name/version.webc, ready to symlinkJoin into an # --include-webc tree. Its .shim drives this packed artifact (what ships), # vs the pkg .shim below which drives the wasmer.toml source dir. webc = let built = pkgs.runCommand "webc-${owner}-${name}-${version}" ({ - passthru.wasix.updateNotes = lib.optional (staleRels != []) { - message = "rels.json has stale keys (${lib.concatMapStringsSep ", " (v: "wasmerPackages.${name} ${v}") staleRels}); nix run .#scripts.update -- --only nixpkgs drops them"; - when = _: _: true; + passthru.wasix = { + inherit publication; + updateNotes = lib.optional (staleRels != []) { + message = "rels.json has stale keys (${lib.concatMapStringsSep ", " (v: "wasmerPackages.${name} ${v}") staleRels}); nix run .#scripts.update -- --only nixpkgs drops them"; + when = _: _: true; + }; }; } // pkgs.lib.optionalAttrs (packagePos != null) {pos = packagePos;}) '' diff --git a/scripts/ci/README.md b/scripts/ci/README.md new file mode 100644 index 00000000..0b0c406b --- /dev/null +++ b/scripts/ci/README.md @@ -0,0 +1,21 @@ +# CI implementation + +`cli.py` is the single local and workflow entry point. `request.py` parses and +normalizes commands, `workspace.py` materializes reproducible cases, +`planner.py` creates the dynamic plan, and `executor.py` runs its phases. +`tasks/` contains task-specific execution and collection. `report/` contains +the generic fragment envelope, renderer, schemas, and GitHub transport. A task +owns its payload and Markdown; the renderer only understands task policy and +the common fragment envelope. + +Formatting is a standalone required validation, not a member of `ciSets.core`. +It runs on candidate materializations without gating build scheduling. Eval-map +job policy distinguishes content-bearing artifacts from validation checks, so +empty success-token outputs never enter content comparison. + +`comment.py` is the authorization adapter for the reusable issue-comment +workflow. It emits a validated `origin.json`; it does not parse a second build +language. See `docs/github-app.md`. + +Generated state lives under `CI_RUN_DIR` (default `.ci-run` locally), never +beside these sources. See `docs/ci.md` for the directory and retention contract. diff --git a/scripts/ci/__init__.py b/scripts/ci/__init__.py new file mode 100644 index 00000000..1b906f34 --- /dev/null +++ b/scripts/ci/__init__.py @@ -0,0 +1 @@ +"""Local-first CI request, execution, and reporting helpers.""" diff --git a/scripts/ci/cli.py b/scripts/ci/cli.py new file mode 100644 index 00000000..b7a65139 --- /dev/null +++ b/scripts/ci/cli.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Public local-first interface for wasinix CI.""" + +import argparse +import json +import os +import shlex +import subprocess +import sys +from pathlib import Path + +CI_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(CI_DIR)) + +from request import RequestError, normalize, parse_command, write_request # noqa: E402 +from dispatch import accept_event # noqa: E402 +from executor import ( # noqa: E402 + execute, + phase_build, + phase_compare, + phase_content, + phase_eval, + phase_spot, + phase_treefmt, + prepare_all, + render_report, +) +from planner import write_plan # noqa: E402 +from workspace import reproduce, write_materialization # noqa: E402 + + +def repo_root(): + p = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], text=True, capture_output=True + ) + if p.returncode != 0: + raise RequestError("CI must run from a git checkout") + return Path(p.stdout.strip()) + + +def load(path): + return json.loads(Path(path).read_text()) + + +def request_command(args, repo): + tokens = shlex.split(args.command_string) if args.command_string else args.command + if tokens and tokens[0] == "--": + tokens = tokens[1:] + request = normalize(parse_command(tokens), repo) + written = write_request(args.out, request) + print(written["requestId"]) + + +def accept_command(args): + written = accept_event(args.event, args.out) + print(written["requestId"]) + + +def prepare_command(args, repo): + request = load(args.request) + if request["action"] == "diff": + cases = {case["id"]: case for case in request["cases"]} + if args.case not in cases: + raise RequestError(f"unknown diff case {args.case!r}") + request = cases[args.case] + elif args.case: + raise RequestError("--case is only valid for a diff request") + write_materialization(repo, request, args.out_dir) + + +def prepare_all_command(args, repo): + matrix = prepare_all(repo, load(args.request), args.run_dir) + print(json.dumps(matrix, sort_keys=True)) + + +def apply_command(args, repo): + reproduce(repo, load(args.request), args.patch) + + +def plan_command(args): + write_plan(args.out, load(args.request)) + + +def run_request(args, repo): + raise SystemExit( + execute( + repo, + load(args.request), + args.run_dir, + runner_local=args.runner_local, + ) + ) + + +def run_public(tokens, repo): + request = normalize(parse_command(tokens), repo) + run_dir = Path(os.environ.get("CI_RUN_DIR", ".ci-run")) + raise SystemExit(execute(repo, request, run_dir)) + + +def phase_case_request(args): + request = load(args.request) + cases = request["cases"] if request["action"] == "diff" else [request] + found = next((case for case in cases if case.get("id", "case") == args.case), None) + if found is None: + raise RequestError(f"unknown case {args.case!r}") + return found + + +def parser(): + ap = argparse.ArgumentParser(prog="ci") + sub = ap.add_subparsers(dest="phase") + request = sub.add_parser("request", help="normalize a public CI command") + request.add_argument("--out", required=True) + request.add_argument("--command-string") + request.add_argument("command", nargs=argparse.REMAINDER) + accept = sub.add_parser("accept", help="validate a repository_dispatch request") + accept.add_argument("--event", required=True) + accept.add_argument("--out", required=True) + prepare = sub.add_parser("prepare", help="materialize one normalized case") + prepare.add_argument("--request", required=True) + prepare.add_argument("--case") + prepare.add_argument("--out-dir", required=True) + prepare_all_parser = sub.add_parser( + "prepare-all", help="materialize every request case" + ) + prepare_all_parser.add_argument("--request", required=True) + prepare_all_parser.add_argument("--run-dir", required=True) + apply = sub.add_parser("apply", help="reproduce a prepared case checkout") + apply.add_argument("--request", required=True) + apply.add_argument("--patch", required=True) + plan = sub.add_parser("plan", help="create a task plan for a request") + plan.add_argument("--request", required=True) + plan.add_argument("--out", required=True) + run = sub.add_parser("run", help="execute a normalized request") + run.add_argument("--request", required=True) + run.add_argument("--run-dir", required=True) + run.add_argument("--runner-local", action="store_true") + for name in ("eval", "spot-task", "treefmt-task"): + phase = sub.add_parser(name) + phase.add_argument("--request", required=True) + phase.add_argument("--case", required=True) + phase.add_argument("--patch", required=True) + phase.add_argument("--run-dir", required=True) + build = sub.add_parser("build-task") + build.add_argument("--request", required=True) + build.add_argument("--case", required=True) + build.add_argument("--patch", required=True) + build.add_argument("--run-dir", required=True) + build.add_argument( + "--name", required=True, choices=["core", "packages", "python", "jobs"] + ) + compare = sub.add_parser("compare") + compare.add_argument("--request", required=True) + compare.add_argument("--run-dir", required=True) + compare.add_argument("--candidate") + content = sub.add_parser("content") + content.add_argument("--request", required=True) + content.add_argument("--run-dir", required=True) + content.add_argument("--candidate", required=True) + report = sub.add_parser("report") + report.add_argument("--run-dir", required=True) + return ap + + +def main(): + if len(sys.argv) > 1 and sys.argv[1] in {"build", "spot", "diff"}: + try: + run_public(sys.argv[1:], repo_root()) + except (OSError, json.JSONDecodeError, RequestError) as error: + print(f"ci: {error}", file=sys.stderr) + raise SystemExit(2) from error + ap = parser() + args = ap.parse_args() + try: + repo = repo_root() + if args.phase == "request": + request_command(args, repo) + elif args.phase == "accept": + accept_command(args) + elif args.phase == "prepare": + prepare_command(args, repo) + elif args.phase == "prepare-all": + prepare_all_command(args, repo) + elif args.phase == "apply": + apply_command(args, repo) + elif args.phase == "plan": + plan_command(args) + elif args.phase == "run": + run_request(args, repo) + elif args.phase == "eval": + phase_eval(repo, phase_case_request(args), args.patch, args.run_dir) + elif args.phase == "treefmt-task": + raise SystemExit( + phase_treefmt(repo, phase_case_request(args), args.patch, args.run_dir) + ) + elif args.phase == "build-task": + raise SystemExit( + phase_build( + repo, + phase_case_request(args), + args.patch, + args.run_dir, + args.name, + ) + ) + elif args.phase == "spot-task": + raise SystemExit( + phase_spot(repo, phase_case_request(args), args.patch, args.run_dir) + ) + elif args.phase == "compare": + raise SystemExit( + phase_compare(repo, load(args.request), args.run_dir, args.candidate) + ) + elif args.phase == "content": + raise SystemExit( + phase_content(repo, load(args.request), args.run_dir, args.candidate) + ) + elif args.phase == "report": + render_report(repo, args.run_dir) + else: + ap.print_help() + raise SystemExit(2) + except (OSError, json.JSONDecodeError, RequestError) as error: + print(f"ci: {error}", file=sys.stderr) + raise SystemExit(2) from error + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/comment.py b/scripts/ci/comment.py new file mode 100644 index 00000000..2b8d1190 --- /dev/null +++ b/scripts/ci/comment.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Authorize a PR comment and prepare a wasinix CI workflow dispatch.""" + +import argparse +import json +import os +import shlex +import sys +import urllib.error +import urllib.request +from pathlib import Path + +from origin import LOGIN, REPOSITORY, validate_origin +from request import RequestError, parse_command + +PREFIX = "@wasinix " +MAX_COMMAND = 4096 +ALLOWED_PERMISSIONS = {"admin", "write"} + + +def require(condition, message): + if not condition: + raise RequestError(f"invalid CI comment: {message}") + + +def github_json(path, token=None): + request = urllib.request.Request(f"https://api.github.com/{path}") + request.add_header("Accept", "application/vnd.github+json") + request.add_header("X-GitHub-Api-Version", "2022-11-28") + token = token or os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + request.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(request) as response: + return json.load(response) + except urllib.error.HTTPError as error: + raise RequestError(f"GitHub API rejected {path}: HTTP {error.code}") from error + + +def command_from_body(body): + require(isinstance(body, str), "comment body must be a string") + require(body.startswith(PREFIX), f"command must start with {PREFIX.strip()}") + command = body[len(PREFIX) :].strip() + require(command, "command is empty") + require(len(command) <= MAX_COMMAND, "command is too long") + require("\n" not in command and "\r" not in command, "command must be one line") + try: + tokens = shlex.split(command) + except ValueError as error: + raise RequestError(f"invalid CI comment: {error}") from error + parsed = parse_command(tokens) + cases = parsed["cases"] if parsed["action"] == "diff" else [parsed] + for case in cases: + execution = case.get("execution", {}) + require(not execution.get("local"), "comment commands cannot use --local") + require(not execution.get("dryRun"), "comment commands cannot use --dry-run") + return command + + +def event_context(event): + require(event.get("action") == "created", "event must create a comment") + issue = event.get("issue") + comment = event.get("comment") + repository = event.get("repository") + require( + isinstance(issue, dict) and issue.get("pull_request"), + "comment is not on a pull request", + ) + require(isinstance(comment, dict), "comment payload is missing") + require(isinstance(repository, dict), "repository payload is missing") + actor = comment.get("user", {}).get("login") + full_name = repository.get("full_name") + pull_request = issue.get("number") + comment_id = comment.get("id") + require( + isinstance(full_name, str) and REPOSITORY.fullmatch(full_name), + "repository must be OWNER/REPO", + ) + require( + isinstance(actor, str) and LOGIN.fullmatch(actor), + "comment author must be a GitHub login", + ) + require( + isinstance(pull_request, int) + and not isinstance(pull_request, bool) + and pull_request > 0, + "pull request number is invalid", + ) + require( + isinstance(comment_id, int) + and not isinstance(comment_id, bool) + and comment_id > 0, + "comment id is invalid", + ) + return { + "repository": full_name, + "pullRequest": pull_request, + "commentId": comment_id, + "actor": actor, + "body": comment.get("body"), + } + + +def prepare(event, api=github_json, allowed_owner=None): + context = event_context(event) + command = command_from_body(context.pop("body")) + repository = context["repository"] + actor = context["actor"] + + permission = api(f"repos/{repository}/collaborators/{actor}/permission") + require( + permission.get("permission") in ALLOWED_PERMISSIONS, + "commenter needs write permission", + ) + pull = api(f"repos/{repository}/pulls/{context['pullRequest']}") + require(pull.get("state") == "open", "pull request is not open") + require( + pull.get("base", {}).get("repo", {}).get("full_name", "").lower() + == str(repository).lower(), + "pull request base repository does not match the event", + ) + origin = validate_origin( + { + "schema": 1, + **context, + "headSha": str(pull.get("head", {}).get("sha", "")).lower(), + }, + allowed_owner, + ) + return { + "schema": 1, + "command": command, + "concurrency": ( + f"command-{origin['repository'].replace('/', '-')}-" + f"pr-{origin['pullRequest']}" + ).lower(), + "origin": origin, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--event", required=True) + parser.add_argument("--allowed-owner", required=True) + parser.add_argument("--out", required=True) + args = parser.parse_args() + try: + event = json.loads(Path(args.event).read_text()) + prepared = prepare(event, allowed_owner=args.allowed_owner) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(prepared, indent=2, sort_keys=True) + "\n") + print(f"accepted {prepared['command']!r} from @{prepared['origin']['actor']}") + except (OSError, json.JSONDecodeError, RequestError) as error: + print(f"comment: {error}", file=sys.stderr) + raise SystemExit(2) from error + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/compare.py b/scripts/ci/compare.py new file mode 100644 index 00000000..ec608aba --- /dev/null +++ b/scripts/ci/compare.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Compare evaluated and built CI cases by stable job name.""" + +import json +import xml.etree.ElementTree as ET +from pathlib import Path + +from planner import requested_jobs, requested_sets +from report.fragment import write_fragment + + +def load(path, default=None): + try: + return json.loads(Path(path).read_text()) + except (OSError, json.JSONDecodeError): + return default + + +def selected(request, mapping): + jobs = set(requested_jobs(request)) + sets = mapping.get("sets", {}) + for name in requested_sets(request): + jobs.update(sets.get(name, [])) + unknown = jobs - set(mapping.get("jobs", {})) - set(mapping.get("errors", {})) + if unknown: + raise ValueError(f"unknown CI job(s): {', '.join(sorted(unknown))}") + return jobs + + +def junit_status(paths): + status = {} + for path in paths: + try: + root = ET.parse(path).getroot() + except (OSError, ET.ParseError): + continue + for case in root.iter("testcase"): + attr = case.get("name", "").strip('"') + if not attr: + continue + failed = case.find("failure") is not None + status[attr] = "failure" if failed else status.get(attr, "success") + return status + + +def identity(info): + if not info or not info.get("version"): + return None + value = str(info["version"]) + if info.get("rel", 1) > 1: + value += f" r{info['rel']}" + return value + + +def named_identity(name, info): + value = identity(info) + return f"{name} {value}" if value else name + + +def compare_cases( + base_request, base_map, base_junit, head_request, head_map, head_junit +): + base_selected = selected(base_request, base_map) + head_selected = selected(head_request, head_map) + base_status = junit_status(base_junit) + head_status = junit_status(head_junit) + both = base_selected & head_selected + + regressions = sorted( + name + for name in both + if base_status.get(name) == "success" and head_status.get(name) == "failure" + ) + fixes = sorted( + name + for name in both + if base_status.get(name) == "failure" and head_status.get(name) == "success" + ) + existing = sorted( + name + for name in both + if base_status.get(name) == "failure" and head_status.get(name) == "failure" + ) + new_eval_errors = sorted( + name + for name in head_selected + if name in head_map.get("errors", {}) and name not in base_map.get("errors", {}) + ) + rebuilt = sorted( + name + for name in both + if base_map.get("jobs", {}).get(name) != head_map.get("jobs", {}).get(name) + and head_map.get("info", {}).get(name, {}).get("rebuildSignal", True) + ) + identity_changed = sorted( + name + for name in both + if identity(base_map.get("info", {}).get(name)) + != identity(head_map.get("info", {}).get(name)) + ) + added = sorted(head_selected - base_selected) + removed = sorted(base_selected - head_selected) + return { + "regressions": regressions, + "fixes": fixes, + "existingFailures": existing, + "newEvalErrors": new_eval_errors, + "rebuilt": rebuilt, + "identityChanged": identity_changed, + "identityTransitions": [ + f"{name}: {identity(base_map.get('info', {}).get(name)) or '?'} -> " + f"{identity(head_map.get('info', {}).get(name)) or '?'}" + for name in identity_changed + ], + "added": added, + "addedIdentities": [ + named_identity(name, head_map.get("info", {}).get(name)) for name in added + ], + "removed": removed, + "removedIdentities": [ + named_identity(name, base_map.get("info", {}).get(name)) for name in removed + ], + } + + +def details(title, values, open_=False): + if not values: + return "" + opened = " open" if open_ else "" + items = "\n".join(f"- `{value}`" for value in values[:250]) + return f"\n{title} ({len(values)})\n\n{items}\n\n\n" + + +def render(result, baseline, candidate): + bad = ( + len(result["regressions"]) + + len(result["newEvalErrors"]) + + int(result.get("caseFailure", False)) + ) + md = f"### `{candidate}` versus `{baseline}`\n\n" + md += ( + f"**{bad} regressions** · {len(result['fixes'])} fixes · " + f"{len(result['rebuilt'])} rebuilt · " + f"{len(result['identityChanged'])} version/rel changes\n" + ) + md += details("Build regressions", result["regressions"], True) + md += details("New evaluation failures", result["newEvalErrors"], True) + if result.get("caseFailure"): + missing = ", ".join(result.get("missingResults", [])) + md += "\nA case did not produce complete build results" + md += f": `{missing}`.\n" if missing else ".\n" + md += details("Fixes", result["fixes"]) + md += details("Existing failures", result["existingFailures"]) + md += details( + "Version or rel changed", + result.get("identityTransitions", result["identityChanged"]), + ) + md += details("Rebuilt", result["rebuilt"]) + md += details("Added", result.get("addedIdentities", result["added"])) + md += details("Removed", result.get("removedIdentities", result["removed"])) + return md + + +def write_comparison(path, *, baseline, candidate, result): + regressions = ( + len(result["regressions"]) + + len(result["newEvalErrors"]) + + int(result.get("caseFailure", False)) + ) + return write_fragment( + path, + task_id=f"compare.{candidate}", + label=f"Compare {candidate}", + kind="comparison", + status="failure" if regressions else "success", + headline=( + f"{regressions} regressions" + if regressions + else f"no regressions · {len(result['fixes'])} fixes" + ), + markdown=render(result, baseline, candidate), + data=result, + ) diff --git a/scripts/ci/dispatch.py b/scripts/ci/dispatch.py new file mode 100644 index 00000000..5c239d81 --- /dev/null +++ b/scripts/ci/dispatch.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Validate normalized requests received through repository_dispatch.""" + +import json +import re +from pathlib import Path + +from request import RequestError, SHA, write_request + +MAX_CASES = 4 +SET_NAMES = {"core", "packages", "python", "all"} +SAFE_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+:-]{0,199}\Z") +SAFE_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}\Z") +KEEP = re.compile(r"-?[A-Za-z0-9][A-Za-z0-9._+-]*(?:,-?[A-Za-z0-9][A-Za-z0-9._+-]*)*\Z") + + +def require(condition, message): + if not condition: + raise RequestError(f"invalid dispatched request: {message}") + + +def exact_keys(value, allowed, context): + extra = set(value) - set(allowed) + require(not extra, f"unknown {context} field(s): {', '.join(sorted(extra))}") + + +def validate_id(value): + if "id" in value: + require( + isinstance(value["id"], str) and SAFE_ID.fullmatch(value["id"]), + "request id must be safe for use as a path component", + ) + + +def validate_source(source): + require(isinstance(source, dict), "source must be an object") + exact_keys(source, {"rev", "patch", "workingTree"}, "source") + require( + SHA.fullmatch(str(source.get("rev", ""))) is not None, + "source.rev must be a commit SHA", + ) + require( + source.get("workingTree") is False, + "working-tree requests cannot be dispatched", + ) + require( + source.get("patch") is None, "callers cannot supply materialization patches" + ) + + +def validate_overrides(overrides): + require( + isinstance(overrides, list) and len(overrides) <= 16, + "overrides must be an array of at most 16 items", + ) + targets = [] + for override in overrides: + require(isinstance(override, dict), "each override must be an object") + exact_keys( + override, + {"target", "kind", "value", "repository", "origin"}, + "override", + ) + target = override.get("target") + require( + isinstance(target, str) and SAFE_NAME.fullmatch(target), + "override target must be a safe attribute name", + ) + require( + override.get("kind") in {"release", "revision"}, "unknown override kind" + ) + value = override.get("value") + require(isinstance(value, str) and value, "override value must be a string") + if override["kind"] == "revision": + require( + SHA.fullmatch(value) is not None, + "revision overrides must use commit SHAs", + ) + targets.append(target) + require(len(targets) == len(set(targets)), "override targets must be unique") + + +def validate_build(case): + require(isinstance(case, dict), "build must be an object") + exact_keys( + case, + {"schema", "action", "id", "source", "selectors", "overrides", "execution"}, + "build", + ) + require(case.get("schema") == 1, "unsupported build schema") + validate_id(case) + require(case.get("action") == "build", "diff cases must be builds") + validate_source(case.get("source")) + selectors = case.get("selectors") + require( + isinstance(selectors, list) and 1 <= len(selectors) <= 32, + "build selectors must contain 1 to 32 items", + ) + seen = [] + for selector in selectors: + require(isinstance(selector, dict), "each selector must be an object") + exact_keys(selector, {"kind", "name"}, "selector") + kind = selector.get("kind") + name = selector.get("name") + require(kind in {"set", "job"}, "unknown selector kind") + require(isinstance(name, str) and name, "selector name must be a string") + if kind == "set": + require(name in SET_NAMES, f"unknown CI set {name!r}") + else: + require( + SAFE_NAME.fullmatch(name), "job selector must be a safe attribute name" + ) + seen.append((kind, name)) + require(len(seen) == len(set(seen)), "selectors must be unique") + require( + not any(name == "all" for _, name in seen) or len(seen) == 1, + "all cannot be combined with other selectors", + ) + validate_overrides(case.get("overrides", [])) + execution = case.get("execution", {}) + require(isinstance(execution, dict), "execution must be an object") + exact_keys(execution, {"local"}, "build execution") + require( + not execution.get("local"), "dispatched builds cannot select a local runner" + ) + + +def validate_spot(request): + exact_keys( + request, + { + "schema", + "action", + "id", + "source", + "targets", + "keep", + "base", + "overrides", + "execution", + }, + "spot", + ) + validate_id(request) + validate_source(request.get("source")) + targets = request.get("targets") + require( + isinstance(targets, list) and 1 <= len(targets) <= 16, + "spot targets must contain 1 to 16 items", + ) + require( + all( + isinstance(value, str) and "." in value and SAFE_NAME.fullmatch(value) + for value in targets + ), + "invalid spot target", + ) + require( + SHA.fullmatch(str(request.get("base", ""))) is not None, + "spot base must be a commit SHA", + ) + validate_overrides(request.get("overrides", [])) + keep = request.get("keep") + require( + keep is None or (isinstance(keep, str) and KEEP.fullmatch(keep)), + "spot keep must be a comma-separated attribute list", + ) + execution = request.get("execution", {}) + require(isinstance(execution, dict), "execution must be an object") + exact_keys(execution, {"local", "dryRun"}, "spot execution") + require( + not execution.get("local"), + "dispatched spot builds cannot select a local runner", + ) + require( + not execution.get("dryRun"), "dry-run spot requests should not be dispatched" + ) + + +def validate_request(request): + require(isinstance(request, dict), "request must be an object") + require(request.get("schema") == 1, "unsupported schema") + action = request.get("action") + require(action in {"build", "spot", "diff"}, "unknown action") + request = json.loads(json.dumps(request)) + request.pop("requestId", None) + if action == "build": + validate_build(request) + elif action == "spot": + validate_spot(request) + else: + exact_keys( + request, + {"schema", "action", "baseline", "contentDiff", "cases"}, + "diff", + ) + require( + isinstance(request.get("contentDiff"), bool), "contentDiff must be boolean" + ) + cases = request.get("cases") + require( + isinstance(cases, list) and 2 <= len(cases) <= MAX_CASES, + f"diff must contain 2 to {MAX_CASES} cases", + ) + ids = [] + for case in cases: + validate_build(case) + case_id = case.get("id") + require( + isinstance(case_id, str) and SAFE_ID.fullmatch(case_id), + "every diff case needs a safe id", + ) + ids.append(case_id) + require(len(ids) == len(set(ids)), "diff case ids must be unique") + require( + request.get("baseline") == ids[0], "the first case must be the baseline" + ) + return request + + +def request_from_event(path): + event = json.loads(Path(path).read_text()) + payload = event.get("client_payload", {}) + request = payload.get("request") + if isinstance(request, str): + request = json.loads(request) + return validate_request(request) + + +def accept_event(event_path, out): + return write_request(out, request_from_event(event_path)) diff --git a/scripts/ci/executor.py b/scripts/ci/executor.py new file mode 100644 index 00000000..6a8feffb --- /dev/null +++ b/scripts/ci/executor.py @@ -0,0 +1,608 @@ +#!/usr/bin/env python3 +"""Execute normalized CI cases with the repository's existing task runners.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +from compare import compare_cases, write_comparison +from planner import requested_jobs, requested_sets, write_plan +from request import RequestError, write_request +from report.fragment import write_fragment +from workspace import reproduced_worktree, write_materialization + + +def run(cmd, *, cwd=None, env=None, check=True): + print(f" $ {' '.join(map(str, cmd))}", file=sys.stderr) + p = subprocess.run([str(v) for v in cmd], cwd=cwd, env=env) + if check and p.returncode != 0: + raise RequestError(f"{cmd[0]} exited {p.returncode}") + return p.returncode + + +def run_logged(cmd, log_path, *, cwd=None, env=None): + print(f" $ {' '.join(map(str, cmd))}", file=sys.stderr) + out = Path(log_path) + out.parent.mkdir(parents=True, exist_ok=True) + with out.open("w") as log: + process = subprocess.Popen( + [str(value) for value in cmd], + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + errors="replace", + ) + assert process.stdout is not None + for line in process.stdout: + sys.stderr.write(line) + log.write(line) + return process.wait() + + +def case_dir(run_dir, case_id): + return Path(run_dir) / "cases" / case_id + + +def task_env(paths, repo, force_local=False): + env = os.environ.copy() + env["CI_RUN_DIR"] = str(paths) + builder = Path(repo) / ".remote-builder" + if builder.exists(): + env["WASINIX_BUILDER"] = str(builder) + if not force_local: + resolver = Path(repo) / "scripts" / "remote-builder.sh" + p = subprocess.run([resolver, "store"], text=True, capture_output=True, env=env) + if p.returncode != 0: + raise RequestError( + "remote builds are the default; configure .remote-builder or pass --local" + ) + env["BUILD_STORE"] = p.stdout.strip() + return env + + +def build_store(repo, request, runner_local=False): + if runner_local or request.get("execution", {}).get("local", False): + return None + env = os.environ.copy() + builder = Path(repo) / ".remote-builder" + if builder.exists(): + env["WASINIX_BUILDER"] = str(builder) + p = subprocess.run( + [Path(repo) / "scripts" / "remote-builder.sh", "store"], + text=True, + capture_output=True, + env=env, + ) + if p.returncode != 0: + raise RequestError("could not resolve the configured remote build store") + return p.stdout.strip() + + +def evaluate(runner_root, worktree, request, paths, fragments): + maps = paths / "maps" + maps.mkdir(parents=True, exist_ok=True) + jobs = maps / "eval-jobs.jsonl" + mapping = maps / "eval-map.json" + markdown = maps / "eval.md" + summary = maps / "eval-summary.json" + script = Path(runner_root) / "scripts" / "ci" / "tasks" / "eval-diff.py" + run( + [ + sys.executable, + script, + "--rev", + request["source"]["rev"], + "--jobs-out", + jobs, + "--map-out", + mapping, + "--md-out", + markdown, + "--summary-out", + summary, + ], + cwd=worktree, + ) + case_id = request.get("id", "case") + run( + [ + sys.executable, + Path(runner_root) / "scripts" / "ci" / "report" / "eval-fragment.py", + "--summary", + summary, + "--markdown", + markdown, + "--id", + f"{case_id}.eval", + "--label", + f"{case_id}: Evaluation", + "--out", + fragments / f"{case_id}.eval.json", + ], + cwd=worktree, + ) + if not mapping.exists(): + raise RequestError(f"{case_id}: evaluation failed") + return json.loads(mapping.read_text()) + + +def select_expression(jobs): + keys = json.dumps({name: None for name in jobs}, sort_keys=True) + encoded = json.dumps(keys) + return f"jobs: builtins.intersectAttrs (builtins.fromJSON {encoded}) jobs" + + +def build_task( + runner_root, + worktree, + request, + paths, + fragments, + name, + mapping, + repo, + runner_local=False, +): + env = task_env( + paths, + repo, + force_local=runner_local or request.get("execution", {}).get("local", False), + ) + case_id = request.get("id", "case") + if name == "jobs": + jobs = requested_jobs(request) + missing = sorted(set(jobs) - set(mapping.get("jobs", {}))) + if missing: + raise RequestError(f"unknown CI job(s): {', '.join(missing)}") + env["CI_SET"] = "jobs" + env["CI_ATTR"] = ".#legacyPackages.x86_64-linux.ciSets.all" + env["CI_SELECT_EXPR"] = select_expression(jobs) + result = paths / "junit" / f"{name}.xml" + result.parent.mkdir(parents=True, exist_ok=True) + env["RESULT_FILE"] = str(result) + status = run( + ["bash", Path(runner_root) / "scripts" / "ci" / "tasks" / "build.sh", name] + if name != "jobs" + else ["bash", Path(runner_root) / "scripts" / "ci" / "tasks" / "build.sh"], + cwd=worktree, + env=env, + check=False, + ) + run( + [ + sys.executable, + Path(runner_root) / "scripts" / "ci" / "report" / "build-fragment.py", + "--junit", + result, + "--jobs", + paths / "maps" / "eval-jobs.jsonl", + "--id", + f"{case_id}.{name}", + "--label", + f"{case_id}: {name.title() if name != 'jobs' else 'Selected jobs'}", + "--logs-dir", + paths / "logs" / name, + "--out", + fragments / f"{case_id}.{name}.json", + ], + cwd=worktree, + ) + return status + + +def run_spot(worktree, request, fragments, runner_local=False): + cmd = ["bash", worktree / "scripts" / "spot.sh", "--base", request["base"]] + if request.get("keep"): + cmd += ["--keep", request["keep"]] + if runner_local or request.get("execution", {}).get("local"): + cmd.append("--local") + if request.get("execution", {}).get("dryRun"): + cmd.append("--dry-run") + cmd += request["targets"] + status = run(cmd, cwd=worktree, check=False) + case_id = request.get("id", "case") + write_fragment( + fragments / f"{case_id}.spot.json", + task_id=f"{case_id}.spot", + label=f"{case_id}: Spot", + kind="spot", + status="success" if status == 0 else "failure", + headline="spot build passed" if status == 0 else "spot build failed", + markdown="Spot is experimental evidence, not a shipping verdict.", + ) + return status + + +def run_treefmt(worktree, request, paths, fragments): + log = paths / "logs" / "treefmt" / "treefmt.log" + status = run_logged( + [ + "nix", + "build", + ".#checks.x86_64-linux.treefmt", + "--no-link", + "--print-build-logs", + "--option", + "accept-flake-config", + "true", + ], + log, + cwd=worktree, + ) + case_id = request.get("id", "case") + markdown = "" + if status: + lines = log.read_text(errors="replace").splitlines() + excerpt = "\n".join(lines[-120:]) + markdown = f"```text\n{excerpt}\n```" if excerpt else "No formatter output." + write_fragment( + fragments / f"{case_id}.treefmt.json", + task_id=f"{case_id}.treefmt", + label=f"{case_id}: Formatting", + kind="validation", + status="success" if status == 0 else "failure", + headline="formatting is clean" + if status == 0 + else "formatting changes required", + markdown=markdown, + ) + return status + + +def execute_case(repo, request, run_dir, *, runner_local=False, validate_treefmt=True): + case_id = request.get("id", "case") + paths = case_dir(run_dir, case_id) + prepared = paths / "prepared" + write_materialization(repo, request, prepared) + materialized = json.loads((prepared / "request.json").read_text()) + materialized["id"] = case_id + fragments = Path(run_dir) / "fragments" + fragments.mkdir(parents=True, exist_ok=True) + statuses = [] + with reproduced_worktree( + repo, materialized, prepared / "materialization.patch" + ) as worktree: + if validate_treefmt: + statuses.append(run_treefmt(worktree, materialized, paths, fragments)) + if materialized["action"] == "spot": + statuses.append(run_spot(worktree, materialized, fragments, runner_local)) + else: + mapping = evaluate(repo, worktree, materialized, paths, fragments) + for name in ("core", "packages", "python"): + if name in requested_sets(materialized): + statuses.append( + build_task( + repo, + worktree, + materialized, + paths, + fragments, + name, + mapping, + repo, + runner_local, + ) + ) + if statuses[-1] and name == "core": + break + if requested_jobs(materialized): + statuses.append( + build_task( + repo, + worktree, + materialized, + paths, + fragments, + "jobs", + mapping, + repo, + runner_local, + ) + ) + return materialized, max(statuses or [0]) + + +def prepare_all(repo, request, run_dir): + run_dir = Path(run_dir).resolve() + run_dir.mkdir(parents=True, exist_ok=True) + request = write_request(run_dir / "request.json", request) + write_plan(run_dir / "plan.json", request) + cases = request["cases"] if request["action"] == "diff" else [request] + prepared_cases = [] + matrix = [] + for case in cases: + case_id = case.get("id", "case") + prepared = case_dir(run_dir, case_id) / "prepared" + write_materialization(repo, case, prepared) + value = json.loads((prepared / "request.json").read_text()) + value["id"] = case_id + (prepared / "request.json").write_text( + json.dumps(value, indent=2, sort_keys=True) + "\n" + ) + prepared_cases.append(value) + sets = requested_sets(value) + matrix.append( + { + "id": case_id, + "rev": value["source"]["rev"], + "treefmt": request["action"] != "diff" + or case_id != request["baseline"], + "eval": value["action"] == "build", + "core": "core" in sets, + "packages": "packages" in sets, + "python": "python" in sets, + "jobs": bool(requested_jobs(value)), + "spot": value["action"] == "spot", + "compare": request["action"] == "diff" + and case_id != request["baseline"], + "content": request["action"] == "diff" + and case_id != request["baseline"] + and request.get("contentDiff", False), + } + ) + prepared_request = json.loads(json.dumps(request)) + if request["action"] == "diff": + prepared_request["cases"] = prepared_cases + else: + prepared_request = prepared_cases[0] + write_request(run_dir / "prepared-request.json", prepared_request) + (run_dir / "matrix.json").write_text(json.dumps(matrix, sort_keys=True) + "\n") + return matrix + + +def phase_eval(repo, request, patch, run_dir): + paths = case_dir(run_dir, request.get("id", "case")) + fragments = Path(run_dir) / "fragments" + fragments.mkdir(parents=True, exist_ok=True) + with reproduced_worktree(repo, request, patch) as worktree: + evaluate(repo, worktree, request, paths, fragments) + + +def phase_treefmt(repo, request, patch, run_dir): + paths = case_dir(run_dir, request.get("id", "case")) + fragments = Path(run_dir) / "fragments" + fragments.mkdir(parents=True, exist_ok=True) + with reproduced_worktree(repo, request, patch) as worktree: + return run_treefmt(worktree, request, paths, fragments) + + +def phase_build(repo, request, patch, run_dir, name): + paths = case_dir(run_dir, request.get("id", "case")) + mapping = json.loads((paths / "maps" / "eval-map.json").read_text()) + fragments = Path(run_dir) / "fragments" + fragments.mkdir(parents=True, exist_ok=True) + with reproduced_worktree(repo, request, patch) as worktree: + return build_task( + repo, + worktree, + request, + paths, + fragments, + name, + mapping, + repo, + runner_local=True, + ) + + +def phase_spot(repo, request, patch, run_dir): + fragments = Path(run_dir) / "fragments" + fragments.mkdir(parents=True, exist_ok=True) + with reproduced_worktree(repo, request, patch) as worktree: + return run_spot(worktree, request, fragments, runner_local=True) + + +def phase_compare(repo, request, run_dir, candidate_filter=None): + if request["action"] != "diff": + raise RequestError("compare phase requires a diff request") + cases = {case["id"]: case for case in request["cases"]} + baseline_id = request["baseline"] + baseline = cases[baseline_id] + base_paths = case_dir(run_dir, baseline_id) + base_map = json.loads((base_paths / "maps" / "eval-map.json").read_text()) + status = 0 + candidates = request["cases"][1:] + if candidate_filter: + candidates = [case for case in candidates if case["id"] == candidate_filter] + if not candidates: + raise RequestError(f"unknown candidate {candidate_filter!r}") + for candidate in candidates: + candidate_id = candidate["id"] + head_paths = case_dir(run_dir, candidate_id) + head_map = json.loads((head_paths / "maps" / "eval-map.json").read_text()) + result = compare_cases( + baseline, + base_map, + junit_paths(base_paths), + candidate, + head_map, + junit_paths(head_paths), + ) + missing = missing_build_results(baseline, base_paths) + missing_build_results( + candidate, head_paths + ) + result["caseFailure"] = bool(missing) + result["missingResults"] = missing + fragment = write_comparison( + Path(run_dir) / "fragments" / f"compare.{candidate_id}.json", + baseline=baseline_id, + candidate=candidate_id, + result=result, + ) + status = max(status, int(fragment["status"] == "failure")) + return status + + +def phase_content(repo, request, run_dir, candidate_id): + if request["action"] != "diff" or not request.get("contentDiff"): + raise RequestError("content phase requires a content-enabled diff request") + cases = {case["id"]: case for case in request["cases"]} + if candidate_id not in cases or candidate_id == request["baseline"]: + raise RequestError(f"unknown candidate {candidate_id!r}") + return content_comparison( + repo, + run_dir, + request, + cases[request["baseline"]], + cases[candidate_id], + runner_local=True, + ) + + +def junit_paths(paths): + return sorted((paths / "junit").glob("*.xml")) + + +def missing_build_results(request, paths): + expected = set(requested_sets(request)) + if requested_jobs(request): + expected.add("jobs") + case_id = request.get("id", "case") + return sorted( + f"{case_id}:{name}" + for name in expected + if not (paths / "junit" / f"{name}.xml").exists() + ) + + +def content_comparison(repo, run_dir, request, baseline, candidate, runner_local): + base_paths = case_dir(run_dir, baseline["id"]) + head_paths = case_dir(run_dir, candidate["id"]) + out = Path(run_dir) / "comparisons" / candidate["id"] / "content" + out.mkdir(parents=True, exist_ok=True) + cmd = [ + sys.executable, + Path(repo) / "scripts" / "ci" / "tasks" / "content-diff.py", + "--left-map", + base_paths / "maps" / "eval-map.json", + "--right-map", + head_paths / "maps" / "eval-map.json", + ] + junits = junit_paths(head_paths) + if junits: + cmd += ["--junit", *junits] + for name in sorted(requested_sets(candidate)): + cmd += ["--built-set", name] + task_id = f"content-diff.{candidate['id']}" + cmd += [ + "--task-id", + task_id, + "--label", + f"Content diff: {candidate['id']}", + "--md-out", + out / "content-diff.md", + "--summary-out", + out / "summary.json", + "--fragment-out", + Path(run_dir) / "fragments" / f"{task_id}.json", + ] + left_store = build_store(repo, baseline, runner_local) + right_store = build_store(repo, candidate, runner_local) + if left_store and left_store == right_store: + cmd += ["--store", left_store] + return run(cmd, cwd=repo, check=False) + + +def execute(repo, request, run_dir, *, runner_local=False): + run_dir = Path(run_dir).resolve() + run_dir.mkdir(parents=True, exist_ok=True) + request = write_request(run_dir / "request.json", request) + write_plan(run_dir / "plan.json", request) + cases = request["cases"] if request["action"] == "diff" else [request] + materialized = {} + statuses = [] + for index, case in enumerate(cases): + case_id = case.get("id", "case") + try: + value, status = execute_case( + repo, + case, + run_dir, + runner_local=runner_local, + validate_treefmt=request["action"] != "diff" or index > 0, + ) + materialized[case_id] = value + if request["action"] != "diff": + statuses.append(status) + except RequestError as error: + if request["action"] != "diff": + statuses.append(1) + print(f"ci: {case_id}: {error}", file=sys.stderr) + if request["action"] == "diff": + baseline_id = request["baseline"] + base_paths = case_dir(run_dir, baseline_id) + for candidate in request["cases"][1:]: + candidate_id = candidate["id"] + head_paths = case_dir(run_dir, candidate_id) + if baseline_id not in materialized or candidate_id not in materialized: + write_fragment( + run_dir / "fragments" / f"compare.{candidate_id}.json", + task_id=f"compare.{candidate_id}", + label=f"Compare {candidate_id}", + kind="comparison", + status="failure", + headline="comparison unavailable", + markdown="A case failed before producing a comparable eval map.", + ) + statuses.append(1) + continue + base = materialized[baseline_id] + base_map = json.loads((base_paths / "maps" / "eval-map.json").read_text()) + head_map = json.loads((head_paths / "maps" / "eval-map.json").read_text()) + result = compare_cases( + base, + base_map, + junit_paths(base_paths), + materialized[candidate_id], + head_map, + junit_paths(head_paths), + ) + missing = missing_build_results(base, base_paths) + missing_build_results( + materialized[candidate_id], head_paths + ) + result["caseFailure"] = bool(missing) + result["missingResults"] = missing + fragment = write_comparison( + run_dir / "fragments" / f"compare.{candidate_id}.json", + baseline=baseline_id, + candidate=candidate_id, + result=result, + ) + statuses.append(1 if fragment["status"] == "failure" else 0) + if request.get("contentDiff"): + content_comparison( + repo, + run_dir, + request, + base, + materialized[candidate_id], + runner_local, + ) + render_report(repo, run_dir) + return max(statuses or [0]) + + +def render_report(repo, run_dir): + report_dir = Path(run_dir) / "report" + report_dir.mkdir(parents=True, exist_ok=True) + run( + [ + sys.executable, + Path(repo) / "scripts" / "ci" / "report" / "render.py", + "--plan", + Path(run_dir) / "plan.json", + "--fragments", + Path(run_dir) / "fragments", + "--fallback-conclusion", + "failure", + "--md-out", + report_dir / "report.md", + "--json-out", + report_dir / "report.json", + ], + cwd=repo, + ) diff --git a/scripts/ci/origin.py b/scripts/ci/origin.py new file mode 100644 index 00000000..337f540b --- /dev/null +++ b/scripts/ci/origin.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Validate the immutable origin of a cross-repository CI command.""" + +import argparse +import json +import re +import sys +from pathlib import Path + +from request import RequestError, SHA + +SCHEMA = 1 +REPOSITORY = re.compile( + r"[A-Za-z0-9](?:[A-Za-z0-9._-]{0,38})/" + r"[A-Za-z0-9](?:[A-Za-z0-9._-]{0,99})\Z" +) +LOGIN = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})\Z") + + +def require(condition, message): + if not condition: + raise RequestError(f"invalid CI command origin: {message}") + + +def validate_origin(value, allowed_owner=None): + require(isinstance(value, dict), "origin must be an object") + allowed = { + "schema", + "repository", + "pullRequest", + "headSha", + "commentId", + "actor", + } + extra = set(value) - allowed + require(not extra, f"unknown field(s): {', '.join(sorted(extra))}") + require(value.get("schema") == SCHEMA, "unsupported schema") + + repository = value.get("repository") + require( + isinstance(repository, str) and REPOSITORY.fullmatch(repository), + "repository must be OWNER/REPO", + ) + owner, name = repository.split("/", 1) + if allowed_owner: + require( + owner.lower() == allowed_owner.lower(), "repository owner is not allowed" + ) + + pull_request = value.get("pullRequest") + comment_id = value.get("commentId") + require( + isinstance(pull_request, int) + and not isinstance(pull_request, bool) + and pull_request > 0, + "pullRequest must be a positive integer", + ) + require( + isinstance(comment_id, int) + and not isinstance(comment_id, bool) + and comment_id > 0, + "commentId must be a positive integer", + ) + require( + isinstance(value.get("headSha"), str) + and SHA.fullmatch(value["headSha"]) + and value["headSha"] == value["headSha"].lower(), + "headSha must be a lowercase commit SHA", + ) + require( + isinstance(value.get("actor"), str) and LOGIN.fullmatch(value["actor"]), + "actor must be a GitHub login", + ) + return { + "schema": SCHEMA, + "repository": f"{owner}/{name}", + "pullRequest": pull_request, + "headSha": value["headSha"], + "commentId": comment_id, + "actor": value["actor"], + } + + +def load_origin(value=None, path=None): + if value: + raw = json.loads(value) + elif path: + raw = json.loads(Path(path).read_text()) + else: + raise RequestError("CI command origin is missing") + return raw + + +def write_origin(path, origin): + out = Path(path) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(origin, indent=2, sort_keys=True) + "\n") + + +def write_token_target(path, github_output, allowed_owner): + origin_path = Path(path) + if not origin_path.exists(): + with Path(github_output).open("a") as output: + output.write("external=false\n") + return + origin = validate_origin(load_origin(path=origin_path), allowed_owner) + owner, repository = origin["repository"].split("/", 1) + with Path(github_output).open("a") as output: + output.write("external=true\n") + output.write(f"owner={owner}\n") + output.write(f"repository={repository}\n") + + +def main(): + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + validate = sub.add_parser("validate") + source = validate.add_mutually_exclusive_group(required=True) + source.add_argument("--value") + source.add_argument("--input") + validate.add_argument("--allowed-owner") + validate.add_argument("--out", required=True) + target = sub.add_parser("token-target") + target.add_argument("--input", required=True) + target.add_argument("--allowed-owner", required=True) + target.add_argument("--github-output", required=True) + args = parser.parse_args() + + try: + if args.command == "validate": + origin = validate_origin( + load_origin(args.value, args.input), args.allowed_owner + ) + write_origin(args.out, origin) + print(f"{origin['repository']}#{origin['pullRequest']}") + else: + write_token_target(args.input, args.github_output, args.allowed_owner) + except (OSError, json.JSONDecodeError, RequestError) as error: + print(f"origin: {error}", file=sys.stderr) + raise SystemExit(2) from error + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/planner.py b/scripts/ci/planner.py new file mode 100644 index 00000000..9b722606 --- /dev/null +++ b/scripts/ci/planner.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Turn normalized CI requests into dynamic task plans.""" + +import json +from pathlib import Path + +SET_ORDER = ("core", "packages", "python") + + +def cases_of(request): + if request["action"] == "diff": + return request["cases"] + return [request] + + +def requested_sets(build): + names = {s["name"] for s in build.get("selectors", []) if s["kind"] == "set"} + if "all" in names: + return set(SET_ORDER) + if names & {"packages", "python"}: + names.add("core") + return names + + +def requested_jobs(build): + return [s["name"] for s in build.get("selectors", []) if s["kind"] == "job"] + + +def plan_of(request): + tasks = [] + order = 0 + case_blocking = request["action"] != "diff" + for case in cases_of(request): + case_id = case.get("id", "case") + if request["action"] != "diff" or case_id != request["baseline"]: + order += 10 + tasks.append( + { + "id": f"{case_id}.treefmt", + "label": f"{case_id}: Formatting", + "kind": "validation", + "order": order, + "blocking": True, + "enabled": True, + } + ) + if case["action"] == "spot": + order += 10 + tasks.append( + { + "id": f"{case_id}.spot", + "label": f"{case_id}: Spot", + "kind": "spot", + "order": order, + "blocking": True, + "enabled": True, + } + ) + continue + order += 10 + tasks.append( + { + "id": f"{case_id}.eval", + "label": f"{case_id}: Evaluation", + "kind": "eval", + "order": order, + "blocking": case_blocking, + "enabled": True, + } + ) + sets = requested_sets(case) + for name in SET_ORDER: + if name not in sets: + continue + order += 10 + tasks.append( + { + "id": f"{case_id}.{name}", + "label": f"{case_id}: {name.title()}", + "kind": "build", + "order": order, + "blocking": case_blocking, + "enabled": True, + } + ) + if requested_jobs(case): + order += 10 + tasks.append( + { + "id": f"{case_id}.jobs", + "label": f"{case_id}: Selected jobs", + "kind": "build", + "order": order, + "blocking": case_blocking, + "enabled": True, + } + ) + if request["action"] == "diff": + for case in request["cases"][1:]: + order += 10 + tasks.append( + { + "id": f"compare.{case['id']}", + "label": f"Compare {case['id']}", + "kind": "comparison", + "order": order, + "blocking": True, + "enabled": True, + } + ) + if request.get("contentDiff"): + order += 10 + tasks.append( + { + "id": f"content-diff.{case['id']}", + "label": f"Content diff: {case['id']}", + "kind": "analysis", + "order": order, + "blocking": False, + "enabled": True, + } + ) + return {"schema": 1, "requestId": request.get("requestId"), "tasks": tasks} + + +def write_plan(path, request): + plan = plan_of(request) + out = Path(path) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n") + return plan diff --git a/scripts/ci-report.py b/scripts/ci/report/build-fragment.py similarity index 76% rename from scripts/ci-report.py rename to scripts/ci/report/build-fragment.py index bf8ddb6e..0bb3318f 100755 --- a/scripts/ci-report.py +++ b/scripts/ci/report/build-fragment.py @@ -1,23 +1,28 @@ #!/usr/bin/env python3 -# Render nix-fast-build's JUnit output and the eval-diff summary into the CI -# report: a one-line status (check run title) in report.json plus markdown -# details. Replaces dorny/test-reporter; test-report.yml turns these into a -# check run and a sticky PR comment. +# Turn one nix-fast-build JUnit result into a generic CI task fragment, retaining +# bounded full logs for direct failures and dependency root causes. # # Tolerates a missing JUnit file (build cancelled before it wrote results) so # the report still carries the eval side. import argparse +import gzip +import hashlib import json -import re import os +import re import subprocess import sys import xml.etree.ElementTree as ET +from pathlib import Path + +from fragment import write_fragment MAX_FAILURES_SHOWN = 25 MAX_LOG_LINES = 30 MAX_LOG_CHARS = 3000 +MAX_ARCHIVED_LOG_BYTES = 20 * 1024 * 1024 +MAX_ARCHIVED_TASK_BYTES = 100 * 1024 * 1024 def parse_junit(path): @@ -41,6 +46,17 @@ def parse_junit(path): return cases +def parse_junits(paths): + cases = [] + found = False + for path in paths: + parsed = parse_junit(path) + if parsed is not None: + found = True + cases.extend(parsed) + return cases if found else None + + def load_jobs_index(jobs_path): # attr -> {drv, position} from the nix-eval-jobs output (--meta) index = {} @@ -166,26 +182,14 @@ def dependency_root_causes(failed, index): if log is not None and not outputs_valid(drv): name = drv.rsplit("/", 1)[-1].split("-", 1)[1][: -len(".drv")] checked[drv] = roots.setdefault( - drv, {"name": name, "log": log, "jobs": []} + drv, {"drv": drv, "name": name, "log": log, "jobs": []} ) if checked[drv] is not None: checked[drv]["jobs"].append(c["attr"]) return sorted(roots.values(), key=lambda r: r["name"]) -def dedupe_notes(fired): - # {attr: [{message, prior, version}]} -> one entry per message; the - # per-profile attrs collapse to a package name - merged = {} - for attr, notes in (fired or {}).items(): - for n in notes: - merged.setdefault(n["message"], {"name": attr.rsplit(".", 1)[-1], **n}) - return sorted(merged.values(), key=lambda n: n["name"]) - - -def title_of(counts, failed, roots, diff, content): - if diff and diff.get("evalFailed"): - return "eval failed" +def title_of(counts, failed, roots): if counts is None: parts = ["build produced no results"] else: @@ -212,21 +216,9 @@ def title_of(counts, failed, roots, diff, content): parts.append(f"all {counts.get('Eval', [0])[0]} jobs ok") built = counts.get("Build", [0, 0]) parts.append(f"{built[0] - built[1]} built") - if diff and diff.get("baseRev"): - parts.append(f"{diff['rebuilt']} rebuilt vs base") - if diff.get("newErrors"): - parts.append(f"{diff['newErrors']} new eval failures") - if content and content.get("pairs"): - parts.append(f"{content['identical']}/{content['pairs']} outputs identical") return " · ".join(parts) -def with_note_count(title, notes): - if notes: - return f"{title} · 📌 {len(notes)} update notes" - return title - - def excerpt(log): # drop nix's internal-json progress lines (FOD logs are full of them) lines = [ @@ -235,16 +227,6 @@ def excerpt(log): return "\n".join(lines)[-MAX_LOG_CHARS:] -def render_notes(notes): - if not notes: - return "" - md = f"\n
📌 Update notes ({len(notes)})\n\n" - for n in notes: - moved = f" ({n['prior']} -> {n['version']})" if n.get("prior") else "" - md += f"- **{n['name']}**{moved}: {n['message']}\n" - return md + "\n
\n" - - def render_md(title, counts, failed, roots): md = f"### Build report\n\n**{title}**\n" if counts is None: @@ -296,32 +278,80 @@ def render_md(title, counts, failed, roots): return md +def archive_logs(logs_dir, failed, roots, index): + if not logs_dir: + return [] + candidates = [] + for case in failed: + if case.get("transitive") or case.get("log") is None: + continue + job = index.get(case["attr"]) + if job: + candidates.append( + { + "attr": case["attr"], + "drv": job["drv"], + "cause": "direct", + "log": case["log"], + } + ) + candidates.extend( + { + "attr": root["name"], + "drv": root["drv"], + "cause": "dependency", + "jobs": root["jobs"], + "log": root["log"], + } + for root in roots + ) + if not candidates: + return [] + + root = Path(logs_dir) + root.mkdir(parents=True, exist_ok=True) + records = [] + archived = 0 + seen = set() + for candidate in candidates: + drv = candidate["drv"] + if drv in seen or archived >= MAX_ARCHIVED_TASK_BYTES: + continue + seen.add(drv) + raw = candidate.pop("log").encode(errors="replace") + limit = min(MAX_ARCHIVED_LOG_BYTES, MAX_ARCHIVED_TASK_BYTES - archived) + truncated = len(raw) > limit + kept = raw[-limit:] + if truncated: + kept = kept.decode(errors="ignore").encode() + archived += len(kept) + name = hashlib.sha256(drv.encode()).hexdigest()[:20] + ".log.gz" + with gzip.open(root / name, "wb", compresslevel=6) as f: + f.write(kept) + records.append( + { + **candidate, + "path": name, + "bytes": len(raw), + "archivedBytes": len(kept), + "truncated": truncated, + } + ) + (root / "manifest.json").write_text(json.dumps({"schema": 1, "logs": records})) + return records + + def main(): ap = argparse.ArgumentParser() - ap.add_argument("--junit", required=True) + ap.add_argument("--junit", required=True, nargs="+") ap.add_argument("--jobs", help="nix-eval-jobs output, for drv paths (nix log)") - ap.add_argument("--diff-summary", help="summary json from eval-diff.py") - ap.add_argument("--content-summary", help="summary json from content-diff.py") - ap.add_argument("--notes", help="fired updateNotes json from nix eval") - ap.add_argument("--md-out", required=True) - ap.add_argument("--json-out", required=True) + ap.add_argument("--id", required=True) + ap.add_argument("--label", required=True) + ap.add_argument("--logs-dir") + ap.add_argument("--out", required=True) args = ap.parse_args() - def load_optional(path): - if not path: - return None - try: - with open(path) as f: - return json.load(f) - except OSError as e: - print(f"WARN: {e}", file=sys.stderr) - return None - - diff = load_optional(args.diff_summary) - content = load_optional(args.content_summary) - notes = dedupe_notes(load_optional(args.notes)) - - cases = parse_junit(args.junit) + cases = parse_junits(args.junit) index = load_jobs_index(args.jobs) if args.jobs: global LOG_CUTOFF @@ -331,7 +361,7 @@ def load_optional(path): pass counts, failed = (None, []) if cases is None else classify(cases, index) roots = dependency_root_causes(failed, index) - title = with_note_count(title_of(counts, failed, roots, diff, content), notes) + title = title_of(counts, failed, roots) # check-run annotations: direct failures anchored at the package # definition (meta.position, which the overlay loader stamps to our @@ -352,12 +382,18 @@ def load_optional(path): } ) - with open(args.json_out, "w") as f: - json.dump( - {"title": title, "failed": len(failed), "annotations": annotations}, f - ) - with open(args.md_out, "w") as f: - f.write(render_md(title, counts, failed, roots) + render_notes(notes)) + logs = archive_logs(args.logs_dir, failed, roots, index) + write_fragment( + args.out, + task_id=args.id, + label=args.label, + kind="build", + status="failure" if failed or counts is None else "success", + headline=title, + markdown=render_md(title, counts, failed, roots), + annotations=annotations, + data={"counts": counts or {}, "failed": len(failed), "logs": logs}, + ) print(title, file=sys.stderr) diff --git a/scripts/ci/report/eval-fragment.py b/scripts/ci/report/eval-fragment.py new file mode 100644 index 00000000..5893b803 --- /dev/null +++ b/scripts/ci/report/eval-fragment.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Turn eval-diff outputs and update notes into the eval task fragment.""" + +import argparse +from pathlib import Path + +from fragment import load_json, write_fragment + + +def render_notes(fired): + notes = {} + for attr, entries in (fired or {}).items(): + for entry in entries: + notes.setdefault( + entry["message"], {"name": attr.rsplit(".", 1)[-1], **entry} + ) + if not notes: + return "" + lines = [f"\n
📌 Update notes ({len(notes)})\n"] + for note in sorted(notes.values(), key=lambda item: item["name"]): + moved = f" ({note['prior']} -> {note['version']})" if note.get("prior") else "" + lines.append(f"- **{note['name']}**{moved}: {note['message']}") + lines.append("\n
\n") + return "\n".join(lines) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--summary", required=True) + ap.add_argument("--markdown", required=True) + ap.add_argument("--notes") + ap.add_argument("--id", default="eval") + ap.add_argument("--label", default="Evaluation") + ap.add_argument("--out", required=True) + args = ap.parse_args() + + summary = load_json(args.summary, {}) + notes = load_json(args.notes, {}) + try: + markdown = Path(args.markdown).read_text() + except OSError: + markdown = "### Rebuild diff\n\nNo rebuild report was produced.\n" + + if summary.get("evalFailed"): + status = "failure" + headline = "evaluation failed" + elif summary.get("baseRev"): + status = "success" + headline = ( + f"{summary.get('changed', summary.get('rebuilt', 0))} of " + f"{summary.get('total', 0)} jobs changed" + ) + else: + status = "success" + headline = f"{summary.get('total', 0)} jobs evaluated" + + write_fragment( + args.out, + task_id=args.id, + label=args.label, + kind="eval", + status=status, + headline=headline, + markdown=markdown + render_notes(notes), + data=summary, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/report/finalize.py b/scripts/ci/report/finalize.py new file mode 100644 index 00000000..185a14f2 --- /dev/null +++ b/scripts/ci/report/finalize.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Synthesize fragments for jobs that failed before their collectors ran.""" + +import argparse +import json +from pathlib import Path + +from fragment import write_fragment + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--plan", required=True) + ap.add_argument("--fragments", required=True) + ap.add_argument("--required-result", required=True) + ap.add_argument("--python-result", required=True) + ap.add_argument("--content-diff-result", required=True) + args = ap.parse_args() + + fragments = Path(args.fragments) + fragments.mkdir(parents=True, exist_ok=True) + try: + plan = json.loads(Path(args.plan).read_text()) + except (OSError, json.JSONDecodeError): + plan = {"tasks": []} + enabled = {task["id"]: task.get("enabled", True) for task in plan["tasks"]} + + def missing(task_id): + return not (fragments / f"{task_id}.json").exists() + + def emit(task_id, label, kind, status, headline): + write_fragment( + fragments / f"{task_id}.json", + task_id=task_id, + label=label, + kind=kind, + status=status, + headline=headline, + ) + + required_failed = args.required_result in {"failure", "cancelled"} + if required_failed: + status = "cancelled" if args.required_result == "cancelled" else "failure" + if missing("core"): + emit("core", "Core", "build", status, "required job ended before reporting") + if missing("packages"): + emit( + "packages", + "Packages", + "build", + "skipped", + "not run because the required job failed", + ) + + if enabled.get("python") and missing("python"): + if args.python_result in {"failure", "cancelled"}: + status = "cancelled" if args.python_result == "cancelled" else "failure" + emit( + "python", "Python", "build", status, "Python job ended before reporting" + ) + elif args.python_result == "skipped" and required_failed: + emit("python", "Python", "build", "skipped", "blocked by required CI") + elif args.python_result == "skipped": + emit( + "python", + "Python", + "build", + "failure", + "Python job was unexpectedly skipped", + ) + + if enabled.get("content-diff") and missing("content-diff"): + if args.content_diff_result in {"failure", "cancelled"}: + emit( + "content-diff", + "Content diff", + "analysis", + "neutral", + "content-diff job ended before reporting", + ) + elif args.content_diff_result == "skipped" and required_failed: + emit( + "content-diff", + "Content diff", + "analysis", + "skipped", + "blocked by required CI", + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/report/fragment.py b/scripts/ci/report/fragment.py new file mode 100644 index 00000000..c33bf706 --- /dev/null +++ b/scripts/ci/report/fragment.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Shared CI report-fragment model and a small fallback-fragment CLI.""" + +import argparse +import json +from pathlib import Path + +SCHEMA = 1 +STATUSES = {"success", "failure", "cancelled", "skipped", "neutral"} + + +def load_json(path, default=None): + if not path: + return default + try: + with open(path) as f: + return json.load(f) + except OSError: + return default + + +def write_fragment( + path, + *, + task_id, + label, + kind, + status, + headline, + markdown="", + annotations=None, + data=None, +): + if status not in STATUSES: + raise ValueError(f"unknown fragment status: {status}") + fragment = { + "schema": SCHEMA, + "id": task_id, + "label": label, + "kind": kind, + "status": status, + "headline": headline, + "markdown": markdown, + "annotations": annotations or [], + "data": data or {}, + } + out = Path(path) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(fragment, sort_keys=True)) + return fragment + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--id", required=True) + ap.add_argument("--label", required=True) + ap.add_argument("--kind", required=True) + ap.add_argument("--status", required=True, choices=sorted(STATUSES)) + ap.add_argument("--headline", required=True) + ap.add_argument("--markdown-file") + ap.add_argument("--annotations-json") + ap.add_argument("--data-json") + ap.add_argument("--out", required=True) + args = ap.parse_args() + + markdown = "" + if args.markdown_file: + try: + markdown = Path(args.markdown_file).read_text() + except OSError: + pass + write_fragment( + args.out, + task_id=args.id, + label=args.label, + kind=args.kind, + status=args.status, + headline=args.headline, + markdown=markdown, + annotations=load_json(args.annotations_json, []), + data=load_json(args.data_json, {}), + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/report/plan.py b/scripts/ci/report/plan.py new file mode 100644 index 00000000..e3c0f595 --- /dev/null +++ b/scripts/ci/report/plan.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Create the event-specific task plan consumed by the unified renderer.""" + +import argparse +import json +from pathlib import Path + + +def boolean(value): + value = value.lower() + if value in {"1", "true", "yes"}: + return True + if value in {"0", "false", "no"}: + return False + raise argparse.ArgumentTypeError(f"expected boolean, got {value!r}") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--python-enabled", type=boolean, required=True) + ap.add_argument("--content-diff-enabled", type=boolean, required=True) + ap.add_argument("--out", required=True) + args = ap.parse_args() + + plan = { + "schema": 1, + "tasks": [ + { + "id": "eval", + "label": "Evaluation", + "kind": "eval", + "order": 10, + "blocking": True, + "enabled": True, + }, + { + "id": "core", + "label": "Core", + "kind": "build", + "order": 20, + "blocking": True, + "enabled": True, + }, + { + "id": "packages", + "label": "Packages", + "kind": "build", + "order": 30, + "blocking": True, + "enabled": True, + }, + { + "id": "python", + "label": "Python", + "kind": "build", + "order": 40, + "blocking": True, + "enabled": args.python_enabled, + }, + { + "id": "content-diff", + "label": "Content diff", + "kind": "analysis", + "order": 50, + "blocking": False, + "enabled": args.content_diff_enabled, + }, + ], + } + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(plan, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/report/post.js b/scripts/ci/report/post.js new file mode 100644 index 00000000..8ee30bdb --- /dev/null +++ b/scripts/ci/report/post.js @@ -0,0 +1,169 @@ +// Idempotently publish the unified CI snapshot as one check run and one sticky +// PR comment. The snapshot is rendered before this transport-only adapter runs. +const fs = require("fs"); +const path = require("path"); + +const read = (file) => { + try { + return fs.readFileSync(file, "utf8"); + } catch { + return null; + } +}; + +module.exports = async ({ github, context, core }) => { + const reportDir = path.join(process.env.CI_RUN_DIR ?? ".ci-run", "report"); + const originText = read( + path.join(process.env.CI_RUN_DIR ?? ".ci-run", "origin.json"), + ); + const origin = originText ? JSON.parse(originText) : null; + const isWorkflowRun = context.eventName === "workflow_run"; + const sourceRun = isWorkflowRun ? context.payload.workflow_run : null; + const sourceRunId = isWorkflowRun ? sourceRun.id : context.runId; + const headSha = origin + ? origin.headSha + : isWorkflowRun + ? sourceRun.head_sha + : context.eventName === "pull_request" + ? context.payload.pull_request.head.sha + : context.eventName === "repository_dispatch" + ? (context.payload.client_payload.head_sha ?? context.sha) + : context.sha; + const report = JSON.parse(read(path.join(reportDir, "report.json")) ?? "{}"); + const body = + read(path.join(reportDir, "report.md")) ?? "No report snapshot found."; + const title = report.title ?? "CI produced no report"; + const complete = report.complete !== false; + const conclusion = ["success", "failure", "cancelled", "neutral"].includes( + report.conclusion, + ) + ? report.conclusion + : "failure"; + const summary = + body.length > 60000 ? body.slice(0, 60000) + "\n\n(truncated)" : body; + + const [owner, repo] = origin + ? origin.repository.split("/", 2) + : [context.repo.owner, context.repo.repo]; + const sameRepository = + owner.toLowerCase() === context.repo.owner.toLowerCase() && + repo.toLowerCase() === context.repo.repo.toLowerCase(); + const annotations = (sameRepository ? (report.annotations ?? []) : []) + .slice(0, 50) + .map((annotation) => ({ + path: annotation.path, + start_line: annotation.line, + end_line: annotation.line, + annotation_level: "failure", + title: annotation.title, + message: annotation.message, + })); + const output = { + title, + summary, + ...(annotations.length ? { annotations } : {}), + }; + const checkName = origin ? "Wasinix CI" : "Per-package status"; + const externalId = origin + ? `wasinix-ci:${context.repo.owner}/${context.repo.repo}:${sourceRunId}` + : `wasinix-ci:${sourceRunId}`; + const checks = await github.paginate(github.rest.checks.listForRef, { + owner, + repo, + ref: headSha, + check_name: checkName, + per_page: 100, + }); + const existing = checks.find((check) => check.external_id === externalId); + const state = complete + ? { status: "completed", conclusion } + : { status: "in_progress" }; + if (existing) { + await github.rest.checks.update({ + owner, + repo, + check_run_id: existing.id, + ...state, + output, + }); + } else { + await github.rest.checks.create({ + owner, + repo, + name: checkName, + head_sha: headSha, + external_id: externalId, + ...state, + output, + }); + } + + let issueNumber; + if (origin) { + issueNumber = origin.pullRequest; + } else if (isWorkflowRun) { + if (sourceRun.event !== "pull_request") return; + let prs = sourceRun.pull_requests; + if (!prs.length) { + const response = + await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner, + repo, + commit_sha: headSha, + }); + prs = response.data.filter((pr) => pr.state === "open"); + } + if (!prs.length) { + core.warning(`no PR found for ${headSha}`); + return; + } + issueNumber = prs[0].number; + } else { + if (context.eventName !== "pull_request") return; + issueNumber = context.payload.pull_request.number; + } + + // A cancelled older run must not replace the current PR's report. + const pr = await github.rest.pulls.get({ + owner, + repo, + pull_number: issueNumber, + }); + if (pr.data.head.sha !== headSha) { + core.info( + `skip stale report for ${headSha}; PR head is ${pr.data.head.sha}`, + ); + return; + } + + const marker = origin + ? `` + : ""; + const metadata = ``; + const runLink = origin + ? `\n[wasinix CI run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${sourceRunId})\n` + : ""; + const commentBody = `${marker}\n${metadata}${runLink}\n${summary}`; + const comments = await github.paginate(github.rest.issues.listComments, { + owner, + repo, + issue_number: issueNumber, + per_page: 100, + }); + const prior = comments.find((comment) => comment.body.startsWith(marker)); + if (prior) { + await github.rest.issues.updateComment({ + owner, + repo, + comment_id: prior.id, + body: commentBody, + }); + } else { + await github.rest.issues.createComment({ + owner, + repo, + issue_number: issueNumber, + body: commentBody, + }); + } +}; diff --git a/scripts/ci/report/render.py b/scripts/ci/report/render.py new file mode 100644 index 00000000..289270e3 --- /dev/null +++ b/scripts/ci/report/render.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Render all currently available task fragments into one CI snapshot.""" + +import argparse +import json +from pathlib import Path + +from fragment import SCHEMA + +STATUS_LABEL = { + "success": "✅ passed", + "failure": "❌ failed", + "cancelled": "⏹ cancelled", + "skipped": "⏭ skipped", + "neutral": "⚠️ advisory failure", + "pending": "⏳ pending", + "deferred": "⏸ deferred", +} + + +def load(path, default): + try: + return json.loads(Path(path).read_text()) + except (OSError, json.JSONDecodeError): + return default + + +def fragments_under(path): + fragments = {} + root = Path(path) + if not root.exists(): + return fragments + for file in sorted(root.rglob("*.json")): + fragment = load(file, {}) + if fragment.get("schema") != SCHEMA or not fragment.get("id"): + continue + fragments[fragment["id"]] = fragment + return fragments + + +def table_cell(value): + return str(value).replace("|", "\\|").replace("\n", " ") + + +def render(plan, fragments, fallback_conclusion): + tasks = sorted(plan.get("tasks", []), key=lambda task: task.get("order", 0)) + if not tasks: + tasks = [ + { + "id": fragment["id"], + "label": fragment.get("label", fragment["id"]), + "order": i, + "blocking": True, + "enabled": True, + } + for i, fragment in enumerate(fragments.values()) + ] + + rows = [] + pending_blocking = [] + failed_blocking = [] + failed_optional = [] + annotations = [] + details = [] + rendered_tasks = [] + + for task in tasks: + task_id = task["id"] + enabled = task.get("enabled", True) + blocking = task.get("blocking", True) + fragment = fragments.get(task_id) + if not enabled: + status = "deferred" + headline = "not scheduled for this event" + elif fragment is None: + status = "pending" + headline = "waiting for task output" + else: + status = fragment.get("status", "failure") + headline = fragment.get("headline", "no summary") + if status not in STATUS_LABEL or status in {"pending", "deferred"}: + headline = f"invalid fragment status: {status!r}" + status = "failure" + annotations.extend(fragment.get("annotations", [])) + if markdown := fragment.get("markdown"): + opened = " open" if status in {"failure", "cancelled"} else "" + details.append( + f"\n{task['label']}: " + f"{headline}\n\n{markdown.rstrip()}\n\n\n" + ) + + if enabled and blocking and status == "pending": + pending_blocking.append(task["label"]) + if enabled and blocking and status in {"failure", "cancelled"}: + failed_blocking.append(task["label"]) + if enabled and not blocking and status in {"failure", "cancelled", "neutral"}: + failed_optional.append(task["label"]) + + policy = "required" if blocking else "advisory" + if not enabled: + policy = "deferred" + rows.append( + f"|{table_cell(task['label'])}|{policy}|{STATUS_LABEL[status]}|" + f"{table_cell(headline)}|" + ) + rendered_tasks.append( + { + "id": task_id, + "status": status, + "blocking": blocking, + "enabled": enabled, + "headline": headline, + } + ) + + if failed_blocking: + conclusion = "failure" + title = f"required CI failed: {', '.join(failed_blocking)}" + elif pending_blocking: + conclusion = None + title = f"CI in progress: {', '.join(pending_blocking)}" + elif tasks or fragments: + conclusion = "success" + title = "required CI passed" + if failed_optional: + title += f" · advisory failure: {', '.join(failed_optional)}" + else: + conclusion = fallback_conclusion or "failure" + title = "CI produced no report fragments" + + # A required failure terminally blocks its downstream tasks, so do not leave + # the check in progress merely because those tasks never emitted fragments. + complete = bool(failed_blocking) or not pending_blocking + md = ( + f"### CI status\n\n**{title}**\n\n" + "|task|policy|status|summary|\n|:--|:--|:--|:--|\n" + + "\n".join(rows) + + "\n" + + "".join(details) + ) + report = { + "schema": 1, + "title": title, + "conclusion": conclusion, + "complete": complete, + "annotations": annotations, + "tasks": rendered_tasks, + } + return md, report + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--plan", required=True) + ap.add_argument("--fragments", required=True) + ap.add_argument("--fallback-conclusion") + ap.add_argument("--md-out", required=True) + ap.add_argument("--json-out", required=True) + args = ap.parse_args() + + md, report = render( + load(args.plan, {}), + fragments_under(args.fragments), + args.fallback_conclusion, + ) + md_out = Path(args.md_out) + json_out = Path(args.json_out) + md_out.parent.mkdir(parents=True, exist_ok=True) + json_out.parent.mkdir(parents=True, exist_ok=True) + md_out.write_text(md) + json_out.write_text(json.dumps(report, sort_keys=True)) + print(report["title"]) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/report/schemas/build-request-v1.json b/scripts/ci/report/schemas/build-request-v1.json new file mode 100644 index 00000000..ddd201dd --- /dev/null +++ b/scripts/ci/report/schemas/build-request-v1.json @@ -0,0 +1,70 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "source": { + "type": "object", + "required": ["rev", "patch", "workingTree"], + "properties": { + "rev": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "patch": {"type": ["string", "null"], "pattern": "^[0-9a-f]{64}$"}, + "workingTree": {"type": "boolean"} + }, + "additionalProperties": false + }, + "override": { + "type": "object", + "required": ["target", "kind", "value"], + "properties": { + "target": {"type": "string", "minLength": 1}, + "kind": {"enum": ["release", "revision"]}, + "value": {"type": "string", "minLength": 1}, + "repository": {"type": "string"}, + "origin": {"type": "string"} + }, + "additionalProperties": false + } + }, + "type": "object", + "required": ["schema", "action", "source", "selectors", "overrides", "execution"], + "properties": { + "schema": {"const": 1}, + "action": {"const": "build"}, + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$"}, + "requestId": {"type": "string", "pattern": "^[0-9a-f]{20}$"}, + "source": {"$ref": "#/$defs/source"}, + "selectors": { + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "type": "object", + "required": ["kind", "name"], + "properties": { + "kind": {"const": "set"}, + "name": {"enum": ["core", "packages", "python", "all"]} + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["kind", "name"], + "properties": { + "kind": {"const": "job"}, + "name": {"type": "string", "minLength": 1} + }, + "additionalProperties": false + } + ] + } + }, + "overrides": {"type": "array", "items": {"$ref": "#/$defs/override"}}, + "execution": { + "type": "object", + "required": ["local"], + "properties": {"local": {"type": "boolean"}}, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/scripts/ci/report/schemas/command-origin-v1.json b/scripts/ci/report/schemas/command-origin-v1.json new file mode 100644 index 00000000..9178f8e4 --- /dev/null +++ b/scripts/ci/report/schemas/command-origin-v1.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "schema", + "repository", + "pullRequest", + "headSha", + "commentId", + "actor" + ], + "properties": { + "schema": {"const": 1}, + "repository": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,38}/[A-Za-z0-9][A-Za-z0-9._-]{0,99}$" + }, + "pullRequest": {"type": "integer", "minimum": 1}, + "headSha": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "commentId": {"type": "integer", "minimum": 1}, + "actor": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9-]{0,38}$" + } + }, + "additionalProperties": false +} diff --git a/scripts/ci/report/schemas/diff-request-v1.json b/scripts/ci/report/schemas/diff-request-v1.json new file mode 100644 index 00000000..58c98ad0 --- /dev/null +++ b/scripts/ci/report/schemas/diff-request-v1.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["schema", "action", "baseline", "contentDiff", "cases"], + "properties": { + "schema": {"const": 1}, + "action": {"const": "diff"}, + "requestId": {"type": "string"}, + "baseline": {"type": "string"}, + "contentDiff": {"type": "boolean"}, + "cases": { + "type": "array", + "minItems": 2, + "maxItems": 4, + "items": {"$ref": "build-request-v1.json"} + } + }, + "additionalProperties": false +} diff --git a/scripts/ci/report/schemas/fragment-v1.json b/scripts/ci/report/schemas/fragment-v1.json new file mode 100644 index 00000000..135caaf6 --- /dev/null +++ b/scripts/ci/report/schemas/fragment-v1.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["schema", "id", "label", "kind", "status", "headline"], + "properties": { + "schema": {"const": 1}, + "id": {"type": "string"}, + "label": {"type": "string"}, + "kind": {"type": "string"}, + "status": { + "enum": ["success", "failure", "cancelled", "skipped", "neutral"] + }, + "headline": {"type": "string"}, + "markdown": {"type": "string"}, + "annotations": {"type": "array"}, + "data": {"type": "object"} + }, + "additionalProperties": false +} diff --git a/scripts/ci/report/schemas/plan-v1.json b/scripts/ci/report/schemas/plan-v1.json new file mode 100644 index 00000000..350971dd --- /dev/null +++ b/scripts/ci/report/schemas/plan-v1.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["schema", "tasks"], + "properties": { + "schema": {"const": 1}, + "requestId": {"type": ["string", "null"]}, + "tasks": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "label", "kind", "order", "blocking", "enabled"], + "properties": { + "id": {"type": "string"}, + "label": {"type": "string"}, + "kind": {"type": "string"}, + "order": {"type": "integer"}, + "blocking": {"type": "boolean"}, + "enabled": {"type": "boolean"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/scripts/ci/report/schemas/spot-request-v1.json b/scripts/ci/report/schemas/spot-request-v1.json new file mode 100644 index 00000000..d3eb2f50 --- /dev/null +++ b/scripts/ci/report/schemas/spot-request-v1.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["schema", "action", "source", "targets", "overrides", "base", "execution"], + "properties": { + "schema": {"const": 1}, + "action": {"const": "spot"}, + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$"}, + "requestId": {"type": "string", "pattern": "^[0-9a-f]{20}$"}, + "source": {"$ref": "build-request-v1.json#/$defs/source"}, + "targets": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 3}}, + "keep": {"type": ["string", "null"]}, + "base": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "overrides": {"type": "array", "items": {"$ref": "build-request-v1.json#/$defs/override"}}, + "execution": { + "type": "object", + "required": ["local", "dryRun"], + "properties": { + "local": {"type": "boolean"}, + "dryRun": {"type": "boolean"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/scripts/ci/request.py b/scripts/ci/request.py new file mode 100644 index 00000000..e409022b --- /dev/null +++ b/scripts/ci/request.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 +"""Parse and normalize the public wasinix CI command language.""" + +import hashlib +import json +import os +import re +import subprocess +import urllib.request +from pathlib import Path + +SCHEMA = 1 +SETS = {"core", "packages", "python", "all"} +SHA = re.compile(r"[0-9a-fA-F]{40}\Z") +PR = re.compile(r"([^/#]+)/([^#]+)#([1-9][0-9]*)\Z") +GITHUB_REMOTE = re.compile(r"github\.com[:/]([^/]+)/([^/.]+)(?:\.git)?\Z") + + +class RequestError(ValueError): + pass + + +def option_value(tokens, i, name): + token = tokens[i] + prefix = name + "=" + if token.startswith(prefix): + value = token[len(prefix) :] + if not value: + raise RequestError(f"{name} needs a value") + return value, i + 1 + if token == name: + if i + 1 >= len(tokens) or tokens[i + 1].startswith("--"): + raise RequestError(f"{name} needs a value") + return tokens[i + 1], i + 2 + return None, i + + +def parse_selector(value): + if value in SETS: + return {"kind": "set", "name": value} + if value.startswith("attr:") and value != "attr:": + return {"kind": "job", "name": value.removeprefix("attr:")} + raise RequestError( + f"unknown build selector {value!r}; expected core/packages/python/all " + "or attr:" + ) + + +def parse_override(value): + target, sep, source = value.partition("=") + if not sep or not target or not source: + raise RequestError( + f"--with expects TARGET=version:VALUE|rev:SHA, got {value!r}" + ) + kind, sep, resolved = source.partition(":") + if not sep or kind not in {"version", "rev"} or not resolved: + raise RequestError(f"invalid override source {source!r}") + if kind == "rev" and not SHA.fullmatch(resolved): + raise RequestError("revision overrides require a 40-character commit SHA") + return { + "target": target, + "kind": "release" if kind == "version" else "revision", + "value": resolved.lower() if kind == "rev" else resolved, + } + + +def parse_build(tokens): + if not tokens or tokens[0] != "build": + raise RequestError("expected a build command") + selectors = [] + overrides = [] + at = "HEAD" + from_pr = None + local = False + i = 1 + while i < len(tokens): + token = tokens[i] + value, nxt = option_value(tokens, i, "--at") + if value is not None: + at, i = value, nxt + continue + value, nxt = option_value(tokens, i, "--with") + if value is not None: + overrides.append(parse_override(value)) + i = nxt + continue + if token == "--from-pr": + from_pr, i = "current", i + 1 + continue + if token.startswith("--from-pr="): + from_pr, i = token.split("=", 1)[1], i + 1 + if not from_pr: + raise RequestError("--from-pr needs a PR context or OWNER/REPO#NUMBER") + continue + if token == "--local": + local, i = True, i + 1 + continue + if token.startswith("--"): + raise RequestError(f"unknown build option {token!r}") + selectors.append(parse_selector(token)) + i += 1 + if not selectors: + raise RequestError("build needs at least one selector") + names = [(s["kind"], s["name"]) for s in selectors] + if len(set(names)) != len(names): + raise RequestError("build selectors must be unique") + if any(s["name"] == "all" for s in selectors) and len(selectors) != 1: + raise RequestError("all cannot be combined with other build selectors") + targets = [o["target"] for o in overrides] + if len(set(targets)) != len(targets): + raise RequestError("override targets must be unique within one build") + return { + "schema": SCHEMA, + "action": "build", + "source": {"ref": at}, + "selectors": selectors, + "overrides": overrides, + "fromPr": from_pr, + "execution": {"local": local}, + } + + +def parse_spot(tokens): + if not tokens or tokens[0] != "spot": + raise RequestError("expected a spot command") + targets = [] + overrides = [] + at = "HEAD" + base = None + keep = None + from_pr = None + local = False + dry_run = False + i = 1 + while i < len(tokens): + token = tokens[i] + matched = False + for name in ("--at", "--base", "--keep", "--with"): + value, nxt = option_value(tokens, i, name) + if value is None: + continue + if name == "--at": + at = value + elif name == "--base": + base = value + elif name == "--keep": + keep = value + else: + overrides.append(parse_override(value)) + i, matched = nxt, True + break + if matched: + continue + if token == "--from-pr": + from_pr, i = "current", i + 1 + continue + if token.startswith("--from-pr="): + from_pr, i = token.split("=", 1)[1], i + 1 + if not from_pr: + raise RequestError("--from-pr needs a PR context or OWNER/REPO#NUMBER") + continue + if token == "--local": + local, i = True, i + 1 + continue + if token == "--dry-run": + dry_run, i = True, i + 1 + continue + if token.startswith("--"): + raise RequestError(f"unknown spot option {token!r}") + if not token.startswith("attr:") or token == "attr:": + raise RequestError("spot targets must use attr:.") + targets.append(token.removeprefix("attr:")) + i += 1 + if not targets: + raise RequestError("spot needs at least one attr:. target") + if len(set(targets)) != len(targets): + raise RequestError("spot targets must be unique") + return { + "schema": SCHEMA, + "action": "spot", + "source": {"ref": at}, + "targets": targets, + "keep": keep, + "base": base, + "overrides": overrides, + "fromPr": from_pr, + "execution": {"local": local, "dryRun": dry_run}, + } + + +def parse_diff(tokens): + content = False + while tokens and tokens[0] == "--content-diff": + content = True + tokens = tokens[1:] + segments = [[]] + for token in tokens: + if token == "--vs": + if not segments[-1]: + raise RequestError("--vs must separate complete build commands") + segments.append([]) + else: + segments[-1].append(token) + if len(segments) < 2 or not segments[-1]: + raise RequestError("diff needs at least two build commands separated by --vs") + cases = [] + for i, segment in enumerate(segments): + build = parse_build(segment) + build["id"] = "baseline" if i == 0 else f"candidate-{i}" + cases.append(build) + return { + "schema": SCHEMA, + "action": "diff", + "baseline": cases[0]["id"], + "contentDiff": content, + "cases": cases, + } + + +def parse_command(tokens): + if not tokens: + raise RequestError("expected build, spot, or diff") + if tokens[0] == "build": + return parse_build(tokens) + if tokens[0] == "spot": + return parse_spot(tokens) + if tokens[0] == "diff": + return parse_diff(tokens[1:]) + raise RequestError(f"unknown CI command {tokens[0]!r}") + + +def git(repo, *args): + p = subprocess.run(["git", "-C", str(repo), *args], text=True, capture_output=True) + if p.returncode != 0: + raise RequestError((p.stderr or p.stdout).strip()) + return p.stdout.strip() + + +def resolve_rev(repo, ref): + return git(repo, "rev-parse", "--verify", f"{ref}^{{commit}}") + + +def github_json(path): + req = urllib.request.Request(f"https://api.github.com/{path}") + req.add_header("Accept", "application/vnd.github+json") + if token := os.environ.get("GITHUB_TOKEN"): + req.add_header("Authorization", f"Bearer {token}") + with urllib.request.urlopen(req) as response: + return json.load(response) + + +def current_pr(): + if origin_path := os.environ.get("WASINIX_CI_ORIGIN"): + origin = json.loads(Path(origin_path).read_text()) + return { + "base": {"repo": {"full_name": origin["repository"]}}, + "head": {"sha": origin["headSha"]}, + "html_url": ( + f"https://github.com/{origin['repository']}/pull/" + f"{origin['pullRequest']}" + ), + } + event_path = os.environ.get("GITHUB_EVENT_PATH") + if not event_path: + raise RequestError("bare --from-pr requires a GitHub pull_request event") + event = json.loads(Path(event_path).read_text()) + pull = event.get("pull_request") + if not pull: + raise RequestError("bare --from-pr requires a GitHub pull_request event") + return pull + + +def resolve_pr(spec): + if spec == "current": + return current_pr() + match = PR.fullmatch(spec) + if not match: + raise RequestError("--from-pr expects OWNER/REPO#NUMBER") + owner, repo, number = match.groups() + return github_json(f"repos/{owner}/{repo}/pulls/{number}") + + +def current_repository(repo): + if value := os.environ.get("GITHUB_REPOSITORY"): + return value.lower() + try: + remote = git(repo, "remote", "get-url", "origin") + except RequestError: + return "" + match = GITHUB_REMOTE.search(remote) + return f"{match.group(1)}/{match.group(2)}".lower() if match else "" + + +def update_sources(repo): + system = git(repo, "rev-parse", "--show-toplevel") + p = subprocess.run( + [ + "nix", + "eval", + "--json", + f"{system}#legacyPackages.x86_64-linux.updateScripts", + ], + text=True, + capture_output=True, + ) + if p.returncode != 0: + raise RequestError(f"could not discover update sources: {p.stderr.strip()}") + sources = {} + for attr, declaration in json.loads(p.stdout).items(): + source = declaration.get("source") + if source and source.get("kind") == "github": + key = f"{source['owner']}/{source['repo']}".lower() + name = declaration.get("name") or attr.rsplit(".", 1)[-1] + sources.setdefault(key, []).append(name) + return sources + + +def apply_pr(request, pull, repo, sources=None): + base_repo = pull["base"]["repo"]["full_name"] + head_sha = pull["head"]["sha"].lower() + current_repo = current_repository(repo) + if current_repo and base_repo.lower() == current_repo: + request["source"] = {"rev": head_sha} + return + sources = update_sources(repo) if sources is None else sources + matches = sources.get(base_repo.lower(), []) + if len(matches) != 1: + found = ", ".join(matches) or "none" + raise RequestError( + f"PR repository {base_repo} maps to {len(matches)} update targets ({found})" + ) + target = matches[0] + if any(o["target"] == target for o in request["overrides"]): + raise RequestError(f"--from-pr duplicates explicit override {target}") + request["overrides"].append( + { + "target": target, + "kind": "revision", + "value": head_sha, + "repository": base_repo, + "origin": pull.get("html_url", ""), + } + ) + + +def normalize_build(request, repo, sources=None): + request = json.loads(json.dumps(request)) + ref = request["source"]["ref"] + request["source"] = { + "rev": resolve_rev(repo, ref), + "patch": None, + "workingTree": ref == "HEAD", + } + if request.get("fromPr"): + apply_pr(request, resolve_pr(request["fromPr"]), repo, sources) + request.pop("fromPr", None) + return request + + +def normalize(request, repo): + if request["action"] == "build": + return normalize_build(request, repo) + if request["action"] == "spot": + out = normalize_build(request, repo) + out["base"] = resolve_rev(repo, out["base"] or out["source"]["rev"]) + return out + if request["action"] == "diff": + sources = None + cases = [] + for case in request["cases"]: + # Share the expensive source discovery only if a case needs it. + if case.get("fromPr") and sources is None: + pull = resolve_pr(case["fromPr"]) + base_repo = pull["base"]["repo"]["full_name"].lower() + current_repo = current_repository(repo) + if base_repo != current_repo: + sources = update_sources(repo) + cases.append(normalize_build(case, repo, sources)) + out = json.loads(json.dumps(request)) + out["cases"] = cases + return out + raise RequestError(f"unsupported action {request['action']!r}") + + +def request_id(request): + value = json.loads(json.dumps(request)) + value.pop("requestId", None) + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest()[:20] + + +def write_request(path, request): + request = json.loads(json.dumps(request)) + request["requestId"] = request_id(request) + out = Path(path) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(request, indent=2, sort_keys=True) + "\n") + return request diff --git a/scripts/ci-build-remote.sh b/scripts/ci/tasks/build-remote.sh similarity index 97% rename from scripts/ci-build-remote.sh rename to scripts/ci/tasks/build-remote.sh index 7091c55c..6e6690bf 100755 --- a/scripts/ci-build-remote.sh +++ b/scripts/ci/tasks/build-remote.sh @@ -23,7 +23,7 @@ usage: $0 [options] [ssh-host] --no-push build only, skip the cache upload (no doppler needed) -i, --ssh-key FILE ssh identity file for the remote -a, --attr ATTR flake attribute to build - (default: legacyPackages..ci) + (default: legacyPackages..ciSets.all) -r, --result-file F copy the JUnit result file back to this local path USAGE exit 64 @@ -55,7 +55,7 @@ while [ $# -gt 0 ]; do shift done # Fall back to the local .remote-builder config for host and key. -resolver="$(dirname "$0")/remote-builder.sh" +resolver="$(dirname "$0")/../../remote-builder.sh" if [ -z "$remote" ]; then remote=$("$resolver" host) || exit $? fi @@ -82,7 +82,7 @@ if [ -z "$attr" ]; then # Deliberately expanded on the remote (unquoted heredoc below): the # attribute must match the *builder's* system, not ours. # shellcheck disable=SC2016 - attr='legacyPackages.$(nix eval --raw --impure --expr builtins.currentSystem).ci' + attr='legacyPackages.$(nix eval --raw --impure --expr builtins.currentSystem).ciSets.all' fi runner="" diff --git a/scripts/ci-build.sh b/scripts/ci/tasks/build.sh similarity index 69% rename from scripts/ci-build.sh rename to scripts/ci/tasks/build.sh index 35b3a0ea..00bc13ed 100755 --- a/scripts/ci-build.sh +++ b/scripts/ci/tasks/build.sh @@ -1,17 +1,26 @@ #!/usr/bin/env bash -# Build every CI package independently, emitting a JUnit report (one test case -# per package). With a signing key present, each package is signed and uploaded +# Build one CI set independently, emitting a JUnit report (one test case per +# package). With a signing key present, each package is signed and uploaded # to the cache as it builds (--copy-to), so a timeout or cancel never loses # built work. Without a key (e.g. fork PRs) it just builds. set -uo pipefail # no -e: keep building past failures +if [ "$#" -gt 1 ]; then + echo "usage: ci-build [set]" >&2 + exit 64 +fi + ENDPOINT="https://1541b1e8a3fc6ad155ce67ef38899700.r2.cloudflarestorage.com" CACHE_STORE="s3://wasinix-cache?region=auto&endpoint=$ENDPOINT&compression=zstd" CACHE_PUB_KEY="wasinix-1:jvsqbOJGsZxMvg97fuyNCWCc+t2nn6uHB47kQCGNmXI=" -CI_ATTR="${CI_ATTR:-.#legacyPackages.$(nix eval --raw --impure --expr 'builtins.currentSystem').ci}" -RESULT_FILE="${RESULT_FILE:-nix-fast-build-result.xml}" +CI_SET="${1:-${CI_SET:-all}}" +CI_ATTR="${CI_ATTR:-.#legacyPackages.$(nix eval --raw --impure --expr 'builtins.currentSystem').ciSets.$CI_SET}" +RUN_DIR="${CI_RUN_DIR:-.ci-run}" +RESULT_FILE="${RESULT_FILE:-$RUN_DIR/junit/$CI_SET.xml}" +JOBS_FILE="${JOBS_FILE:-$RUN_DIR/maps/eval-jobs.jsonl}" +mkdir -p "$(dirname "$RESULT_FILE")" COPY_ARGS=() if [ -n "${NIX_SIGNING_KEY:-}" ]; then @@ -30,22 +39,43 @@ else echo "No signing key; building without cache upload." fi +STORE_ARGS=() +if [ -n "${BUILD_STORE:-}" ]; then + STORE_ARGS=(--store "$BUILD_STORE") +fi +SELECT_ARGS=() +if [ -n "${CI_SELECT_EXPR:-}" ]; then + SELECT_ARGS=(--select "$CI_SELECT_EXPR") +fi + # --copy-to pushes only runtime closures, so build-only deps never reach the # cache. Capture the needed builds before building (afterwards they would read # "local", not "notBuilt"); they are pushed after the build. JOBS_FILE reuses -# the eval from the rebuild-diff step (scripts/eval-diff.py) instead of +# the eval from the rebuild-diff step (tasks/eval-diff.py) instead of # evaling a second time. # Eval parallelism. One worker per core is right on a dedicated runner, but # every worker registers drvs through the one nix-daemon: on a shared builder # the bottleneck is daemon/SQLite contention, not CPU, so callers there set a -# small EVAL_WORKERS (ci-build-remote.sh). +# small EVAL_WORKERS (tasks/build-remote.sh). EVAL_WORKERS="${EVAL_WORKERS:-$(nproc)}" MAX_JOBS="${MAX_JOBS:-$(nproc)}" PUSH_DRVS="" if [ -n "${NIX_SIGNING_KEY:-}" ]; then - if [ -n "${JOBS_FILE:-}" ] && [ -s "$JOBS_FILE" ]; then - PUSH_DRVS=$(jq -r '.neededBuilds[]?' "$JOBS_FILE" | sort -u) + if [ -s "$JOBS_FILE" ]; then + # The rebuild diff evaluates ciSets.all once. Restrict that shared jobs file + # to this set, or the core build would realise build-only deps from later sets. + set_attrs=$( + nix eval "$CI_ATTR" --apply builtins.attrNames --json \ + --option accept-flake-config true + ) + PUSH_DRVS=$( + jq -r --argjson attrs "$set_attrs" ' + (.attrPath | join(".")) as $attr + | select($attrs | index($attr)) + | .neededBuilds[]? + ' "$JOBS_FILE" | sort -u + ) else PUSH_DRVS=$( nix-eval-jobs \ @@ -72,6 +102,8 @@ nix-fast-build \ --result-file "$RESULT_FILE" \ --result-format junit \ --option accept-flake-config true \ + "${STORE_ARGS[@]}" \ + "${SELECT_ARGS[@]}" \ "${COPY_ARGS[@]}" status=$? diff --git a/scripts/content-diff.py b/scripts/ci/tasks/content-diff.py similarity index 59% rename from scripts/content-diff.py rename to scripts/ci/tasks/content-diff.py index 7058633e..96e60aa5 100755 --- a/scripts/content-diff.py +++ b/scripts/ci/tasks/content-diff.py @@ -17,10 +17,15 @@ # builds skipped by --skip-cached. import argparse +import itertools import json import subprocess import sys import xml.etree.ElementTree as ET +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from report.fragment import write_fragment CACHE_URL = "https://nix-cache.wasix.org" CACHE_PUB_KEY = "wasinix-1:jvsqbOJGsZxMvg97fuyNCWCc+t2nn6uHB47kQCGNmXI=" @@ -30,10 +35,6 @@ NAR_SIZE_CAP = 256 * 1024 * 1024 LIST_CAP = 250 -# Source-tree-tracking drvs rebuild on every diff and their pass output is -# constant; no content signal (matches eval-diff.py). -TREE_TRACKING = ("checks.treefmt",) - def log(msg): print(msg, file=sys.stderr) @@ -60,32 +61,37 @@ def basename(path): return path.rsplit("/", 1)[-1] -def failed_attrs(junit_path): +def failed_attrs(junit_paths): # Attrs whose build (or eval) failed have no new output to compare. - if not junit_path: - return set() - try: - root = ET.parse(junit_path).getroot() - except (OSError, ET.ParseError) as e: - log(f"WARN: no junit ({e})") + if not junit_paths: return set() - return { - tc.get("name", "").strip('"') - for tc in root.iter("testcase") - if tc.find("failure") is not None and tc.get("classname") != "Upload" - } + failed = set() + for path in junit_paths: + try: + root = ET.parse(path).getroot() + except (OSError, ET.ParseError) as e: + log(f"WARN: no junit {path} ({e})") + continue + failed.update( + tc.get("name", "").strip('"') + for tc in root.iter("testcase") + if tc.find("failure") is not None and tc.get("classname") != "Upload" + ) + return failed def self_referential(path, info): return basename(path) in info.get("references", []) -def normalize_pair(old, new): +def normalize_pair(old, new, store=None): # -> (identical, reason|None). Realise both (old substitutes from the # cache), then compare with self-references rewritten to content hashes. # explicit cache: raw nix-store does not read the flake's nixConfig - r = subprocess.run( - [ + if store: + command = ["nix", "copy", "--from", store, old, new] + else: + command = [ "nix-store", "--realise", old, @@ -96,10 +102,8 @@ def normalize_pair(old, new): "--option", "extra-trusted-public-keys", CACHE_PUB_KEY, - ], - capture_output=True, - text=True, - ) + ] + r = subprocess.run(command, capture_output=True, text=True) if r.returncode != 0: return None, f"realise failed: {r.stderr.strip().splitlines()[-1][:120]}" p = subprocess.run( @@ -113,20 +117,25 @@ def normalize_pair(old, new): return rw[old] == rw[new], None -def compare_all(pairs): - olds = path_infos([p["old"] for p in pairs], store=CACHE_URL) - news_local = path_infos([p["new"] for p in pairs]) - # --skip-cached builds may exist only in the cache, not locally - missing = [p["new"] for p in pairs if not news_local.get(basename(p["new"]))] - news_remote = path_infos(missing, store=CACHE_URL) if missing else {} +def available_infos(paths, store=None): + local = path_infos(paths) + missing = [path for path in paths if not local.get(basename(path))] + remote = path_infos(missing, store=store or CACHE_URL) if missing else {} + if store: + still_missing = [path for path in missing if not remote.get(basename(path))] + remote.update(path_infos(still_missing, store=CACHE_URL)) + return {**local, **remote} + + +def compare_all(pairs, store=None): + olds = available_infos([p["old"] for p in pairs], store) + news = available_infos([p["new"] for p in pairs], store) identical, changed, skipped = [], [], [] normalized = 0 for p in pairs: old_info = olds.get(basename(p["old"])) - new_info = news_local.get(basename(p["new"])) or news_remote.get( - basename(p["new"]) - ) + new_info = news.get(basename(p["new"])) if p["old"] == p["new"]: identical.append(p) elif old_info is None or new_info is None: @@ -144,7 +153,7 @@ def compare_all(pairs): skipped.append((p, f"normalize cap ({MAX_NORMALIZE}) reached")) else: normalized += 1 - same, reason = normalize_pair(p["old"], p["new"]) + same, reason = normalize_pair(p["old"], p["new"], store) if same is None: skipped.append((p, reason)) elif same: @@ -170,8 +179,8 @@ def label(p): return f"`{p['attr']}{out}`" -def render(pairs, identical, changed, skipped): - md = "### Content diff\n\n" +def render(pairs, identical, changed, skipped, title="Content diff"): + md = f"### {title}\n\n" md += ( f"Of **{len(pairs)}** rebuilt outputs: **{len(identical)}** bit-identical" f" · **{len(changed)}** changed · {len(skipped)} not comparable\n\n" @@ -186,39 +195,100 @@ def render(pairs, identical, changed, skipped): return md +def content_jobs(base, head, allowed=None): + moved = sorted( + attr + for attr in head["jobs"].keys() & base["jobs"].keys() + if head["jobs"][attr] != base["jobs"][attr] + and (allowed is None or attr in allowed) + ) + included = [ + attr + for attr in moved + if head.get("info", {}) + .get(attr, {}) + .get("contentDiff", not attr.startswith("checks.")) + ] + return included, len(moved) - len(included) + + def main(): ap = argparse.ArgumentParser() - ap.add_argument("--base-map", required=True) - ap.add_argument("--head-map", required=True) - ap.add_argument("--junit", help="skip attrs whose build failed") + ap.add_argument("--left-map", required=True) + ap.add_argument("--right-map", required=True) + ap.add_argument("--junit", nargs="+", help="skip attrs whose build failed") + ap.add_argument( + "--built-set", + action="append", + default=[], + help="only compare jobs belonging to this CI set (repeatable)", + ) ap.add_argument("--md-out", required=True) ap.add_argument("--summary-out", required=True) + ap.add_argument("--fragment-out") + ap.add_argument("--task-id", default="content-diff") + ap.add_argument("--label", default="Content diff") + ap.add_argument("--store", help="store containing both case outputs") args = ap.parse_args() - def done(md, summary): - with open(args.md_out, "w") as f: - f.write(md) - with open(args.summary_out, "w") as f: - json.dump(summary, f) + def done(md, summary, status="success", headline=None): + md_out = Path(args.md_out) + summary_out = Path(args.summary_out) + md_out.parent.mkdir(parents=True, exist_ok=True) + summary_out.parent.mkdir(parents=True, exist_ok=True) + md_out.write_text(md) + summary_out.write_text(json.dumps(summary)) + if headline is None: + headline = ( + f"{summary.get('identical', 0)}/{summary.get('pairs', 0)} outputs identical" + if summary.get("pairs") + else "nothing to compare" + ) + if args.fragment_out: + write_fragment( + args.fragment_out, + task_id=args.task_id, + label=args.label, + kind="analysis", + status=status, + headline=headline, + markdown=md, + data=summary, + ) try: - with open(args.base_map) as f: + with open(args.left_map) as f: base = json.load(f) except OSError: - return done("### Content diff\n\nSkipped: no base eval map.\n", {}) + return done( + "### Content diff\n\nSkipped: no base eval map.\n", + {}, + status="neutral", + headline="skipped: no base eval map", + ) try: - with open(args.head_map) as f: + with open(args.right_map) as f: head = json.load(f) except OSError: # eval failed: eval-diff.py wrote no head map - return done("### Content diff\n\nSkipped: no head eval map.\n", {}) + return done( + "### Content diff\n\nSkipped: no head eval map.\n", + {}, + status="neutral", + headline="skipped: no head eval map", + ) failed = failed_attrs(args.junit) - moved = sorted( - a - for a in head["jobs"].keys() & base["jobs"].keys() - if head["jobs"][a] != base["jobs"][a] and a not in TREE_TRACKING - ) + allowed = None + if args.built_set: + sets = head.get("sets", {}) + missing = [name for name in args.built_set if name not in sets] + if missing: + log(f"WARN: CI set membership missing for {', '.join(missing)}") + allowed = set( + itertools.chain.from_iterable(sets.get(name, []) for name in args.built_set) + ) + moved, excluded = content_jobs(base, head, allowed) rebuilt = [a for a in moved if a not in failed] not_built = len(moved) - len(rebuilt) pairs = [ @@ -233,30 +303,39 @@ def done(md, summary): if not_built else "" ) + excluded_note = ( + f"\n{excluded} validation jobs were excluded from content comparison.\n" + if excluded + else "" + ) if not pairs: return done( - f"### Content diff\n\nNothing built to compare.\n{not_built_note}", + f"### Content diff\n\nNothing built to compare.\n{not_built_note}{excluded_note}", { "pairs": 0, "identical": 0, "changed": 0, "skipped": 0, "notBuilt": not_built, + "excluded": excluded, }, ) - identical, changed, skipped = compare_all(pairs) + identical, changed, skipped = compare_all(pairs, args.store) log( f"{len(pairs)} pairs: {len(identical)} identical, {len(changed)} changed, {len(skipped)} skipped" ) done( - render(pairs, identical, changed, skipped) + not_built_note, + render(pairs, identical, changed, skipped, args.label) + + not_built_note + + excluded_note, { "pairs": len(pairs), "identical": len(identical), "changed": len(changed), "skipped": len(skipped), "notBuilt": not_built, + "excluded": excluded, }, ) diff --git a/scripts/eval-diff.py b/scripts/ci/tasks/eval-diff.py similarity index 68% rename from scripts/eval-diff.py rename to scripts/ci/tasks/eval-diff.py index e1fd3ab9..20501bc8 100755 --- a/scripts/eval-diff.py +++ b/scripts/ci/tasks/eval-diff.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -# Rebuild diff for CI. Evals the ci job set with nix-eval-jobs (attr -> +# Rebuild diff for CI. Evals ciSets.all with nix-eval-jobs (attr -> # drvPath), fetches the same map for a base commit from the cache bucket, and -# renders a markdown summary of what this change rebuilds. ci.yml uploads the -# map on pushes to main and reuses the raw eval for ci-build.sh, so the whole +# renders a markdown summary of changed build identities. build.yml uploads the +# map on pushes to main and reuses the raw eval for tasks/build.sh, so the whole # thing costs one eval per run. # -# Only the drv-level view: meta/passthru-only changes do not move drvPaths and -# stay invisible here (that is the point; see CLAUDE.md "Checking your work"). +# Drv paths identify rebuilds; explicit version/rel identities also surface +# publication-only changes. Other meta/passthru-only changes stay invisible. # # Usage: # eval-diff.py --jobs-out eval-jobs.jsonl --map-out eval-map.json \ @@ -30,31 +30,84 @@ # A drv move under these prefixes rebuilds the world downstream. MASS_REBUILD_PREFIXES = ("toolchain.",) -# Drvs that hash the whole source tree: they move on every diff and carry no -# rebuild signal. Kept as CI jobs, excluded from the diff (content-diff.py -# excludes them too). -TREE_TRACKING = ("checks.treefmt",) - def log(msg): print(msg, file=sys.stderr) -def default_flake(): - system = subprocess.run( +def current_system(): + return subprocess.run( ["nix", "eval", "--raw", "--impure", "--expr", "builtins.currentSystem"], check=True, text=True, capture_output=True, ).stdout.strip() - return f".#legacyPackages.{system}.ci" + + +def default_flake(): + system = current_system() + return f".#legacyPackages.{system}.ciSets.all" + + +def default_sets_flake(): + return f".#legacyPackages.{current_system()}.ciSets" + + +def default_info_flake(): + return f".#legacyPackages.{current_system()}.ciJobInfo" + + +def eval_info(flake): + p = subprocess.run( + [ + "nix", + "eval", + flake, + "--json", + "--option", + "accept-flake-config", + "true", + ], + capture_output=True, + text=True, + ) + if p.returncode != 0: + log(f"WARN: could not evaluate CI job identities: {p.stderr.strip()}") + return {} + return json.loads(p.stdout) + + +def eval_sets(flake): + expr = ( + "sets: builtins.mapAttrs (_: jobs: builtins.attrNames jobs) " + '(removeAttrs sets ["all"])' + ) + p = subprocess.run( + [ + "nix", + "eval", + flake, + "--apply", + expr, + "--json", + "--option", + "accept-flake-config", + "true", + ], + capture_output=True, + text=True, + ) + if p.returncode != 0: + log(f"WARN: could not evaluate CI set membership: {p.stderr.strip()}") + return {} + return json.loads(p.stdout) def eval_jobs(flake, jobs_path): # nix-eval-jobs comes from PATH (the .#scripts.rebuild-diff wrapper and the # devShell both pin it to the locked nixpkgs); `nix run nixpkgs#` would # fetch and unpack the registry's channel tarball on every CI run. - # --check-cache-status so ci-build.sh can reuse this eval for its + # --check-cache-status so tasks/build.sh can reuse this eval for its # build-dep push list. Returns the nix error on a top-level eval failure # (broken flake): that becomes report content, not a step crash; the # build step fails the job on the same error. @@ -63,13 +116,13 @@ def eval_jobs(flake, jobs_path): "--flake", flake, "--check-cache-status", - # one worker per core: the ci key set is cheap to compute (flake.nix + # one worker per core: the CI key set is cheap to compute (flake.nix # reads meta, not drvPath), so workers share little warmup and # parallelize both instantiation and the cache-status lookups. # nix-eval-jobs defaults to a single worker. "--workers", str(os.cpu_count() or 1), - # meta rides along for ci-report.py: meta.position anchors failure + # meta rides along for report/build-fragment.py: meta.position anchors failure # annotations at the package definition "--meta", "--option", @@ -103,7 +156,7 @@ def load_map_from_jobs(jobs_path, rev): errors[attr] = obj["error"].splitlines()[0] else: jobs[attr] = obj["drvPath"] - # output paths feed content-diff.py (rebuilt vs actually changed) + # output paths feed tasks/content-diff.py (rebuilt vs actually changed) outputs[attr] = obj.get("outputs") or {} return { "schema": MAP_SCHEMA, @@ -137,11 +190,29 @@ def fetch_base_map(revs, base_map_path): return None -def section(title, attrs): +def identity(info): + if not info or not info.get("version"): + return None + rendered = str(info["version"]) + if info.get("rel", 1) > 1: + rendered += f" r{info['rel']}" + return rendered + + +def item(attr, base, head, transition=False): + prior = identity(base.get("info", {}).get(attr)) + current = identity(head.get("info", {}).get(attr)) + if transition and prior and current and prior != current: + return f"- `{attr}`: {prior} → {current}" + version = current or prior + return f"- `{attr}`{f' {version}' if version else ''}" + + +def section(title, attrs, base, head, transition=False): if not attrs: return "" shown = sorted(attrs)[:LIST_CAP] - lines = "\n".join(f"- `{a}`" for a in shown) + lines = "\n".join(item(a, base, head, transition) for a in shown) more = len(attrs) - len(shown) if more: lines += f"\n- ... and {more} more" @@ -150,12 +221,23 @@ def section(title, attrs): def diff_of(base, head): both = head["jobs"].keys() & base["jobs"].keys() + rebuilt = { + a + for a in both + if head["jobs"][a] != base["jobs"][a] + and head.get("info", {}).get(a, {}).get("rebuildSignal", True) + } + identity_changed = { + a + for a in both + if identity(base.get("info", {}).get(a)) is not None + and identity(head.get("info", {}).get(a)) is not None + and identity(base["info"][a]) != identity(head["info"][a]) + } return { - "rebuilt": sorted( - a - for a in both - if head["jobs"][a] != base["jobs"][a] and a not in TREE_TRACKING - ), + "changed": sorted(rebuilt | identity_changed), + "rebuilt": sorted(rebuilt), + "identityChanged": sorted(identity_changed), "added": sorted(head["jobs"].keys() - base["jobs"].keys()), "removed": sorted(base["jobs"].keys() - head["jobs"].keys()), "newErrors": { @@ -166,15 +248,16 @@ def diff_of(base, head): def render(base, head): d = diff_of(base, head) - rebuilt, added, removed = d["rebuilt"], d["added"], d["removed"] + changed, rebuilt = d["changed"], d["rebuilt"] + added, removed = d["added"], d["removed"] new_errors = d["newErrors"] total = len(head["jobs"]) md = f"### Rebuild diff vs `{base['rev'][:12]}`\n\n" - if not (rebuilt or added or removed or new_errors): - return md + "No rebuilds.\n" + if not (changed or added or removed or new_errors): + return md + "No changes.\n" md += ( - f"**{len(rebuilt)}** of **{total}** jobs rebuild" + f"**{len(changed)}** of **{total}** jobs changed" f" · {len(added)} added · {len(removed)} removed" f" · {len(new_errors)} new eval failures\n" ) @@ -186,9 +269,9 @@ def render(base, head): f"{', ...' if len(mass) > 5 else ''}): everything downstream rebuilds.\n" ) - md += section("Rebuilt", rebuilt) - md += section("Added", added) - md += section("Removed", removed) + md += section("Changed", changed, base, head, transition=True) + md += section("Added", added, base, head) + md += section("Removed", removed, base, head) if new_errors: lines = "\n".join( f"- `{a}`: {e}" for a, e in sorted(new_errors.items())[:LIST_CAP] @@ -203,16 +286,18 @@ def render(base, head): def main(): ap = argparse.ArgumentParser() ap.add_argument("--flake", default=None) + ap.add_argument("--sets-flake", default=None) + ap.add_argument("--info-flake", default=None) ap.add_argument("--rev", default=None, help="rev recorded in the map") ap.add_argument("--jobs", help="existing nix-eval-jobs output, skips the eval") ap.add_argument("--jobs-out", help="where to write the nix-eval-jobs output") ap.add_argument("--map-out", required=True) ap.add_argument("--md-out", required=True) - ap.add_argument("--summary-out", help="counts as json, for ci-report.py") + ap.add_argument("--summary-out", help="counts as json, for the eval fragment") ap.add_argument("--base-rev", nargs="*", default=[]) ap.add_argument("--base-map", help="local base map file, for testing") ap.add_argument( - "--base-map-out", help="save the fetched base map, for content-diff.py" + "--base-map-out", help="save the fetched base map for tasks/content-diff.py" ) ap.add_argument( "--note-versions", help="updateNotes.versions json, published in the map" @@ -251,6 +336,8 @@ def main(): ).stdout.strip() ) head = load_map_from_jobs(jobs_path, rev) + head["sets"] = eval_sets(args.sets_flake or default_sets_flake()) + head["info"] = eval_info(args.info_flake or default_info_flake()) if args.note_versions: try: with open(args.note_versions) as f: diff --git a/scripts/publish-eval-map.sh b/scripts/ci/tasks/publish-eval-map.sh similarity index 77% rename from scripts/publish-eval-map.sh rename to scripts/ci/tasks/publish-eval-map.sh index e0c509a8..6b5b5a80 100644 --- a/scripts/publish-eval-map.sh +++ b/scripts/ci/tasks/publish-eval-map.sh @@ -4,9 +4,11 @@ # GITHUB_SHA. set -euo pipefail +eval_map="${EVAL_MAP:-${CI_RUN_DIR:-.ci-run}/maps/eval-map.json}" + # absent when the eval failed; never publish a broken base -if [ -f eval-map.json ]; then - aws s3 cp --no-progress eval-map.json \ +if [ -f "$eval_map" ]; then + aws s3 cp --no-progress "$eval_map" \ "s3://wasinix-cache/eval-maps/$GITHUB_SHA.json" \ --endpoint-url https://1541b1e8a3fc6ad155ce67ef38899700.r2.cloudflarestorage.com else diff --git a/scripts/ci/tasks/rebuild-diff.sh b/scripts/ci/tasks/rebuild-diff.sh new file mode 100644 index 00000000..9def5227 --- /dev/null +++ b/scripts/ci/tasks/rebuild-diff.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Eval the CI job map (attr -> drvPath), diff it against the base branch's +# published map to surface what this change rebuilds, and emit the update notes. +# Run in CI (build.yml) via `nix run .#scripts.rebuild-diff`, which provides +# python3 and nix-eval-jobs. Informational, so failures fall back to empty +# rather than aborting. +# Reads GHA env: GITHUB_SHA, GITHUB_STEP_SUMMARY, and BASE_REF (set by the +# workflow from the pull_request / merge_group base). +set -uo pipefail + +ci_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +run_dir="${CI_RUN_DIR:-.ci-run}" +maps_dir="$run_dir/maps" +fragments_dir="$run_dir/fragments" +mkdir -p "$maps_dir" "$fragments_dir" + +candidates=() +if [ -n "${BASE_REF:-}" ]; then + # walk back: the newest base commits may not have published yet + git fetch --quiet --depth=30 origin "$BASE_REF" + mapfile -t candidates < <(git rev-list -n 30 FETCH_HEAD) +fi + +# update notes: current versions ride in the map; the base map's copy comes back +# as the `prior` side of each note's predicate +sys=$(nix eval --raw --impure --expr 'builtins.currentSystem') +nix eval --json ".#legacyPackages.$sys.updateNotes.versions" \ + --option accept-flake-config true >"$maps_dir/note-versions.json" || + echo '{}' >"$maps_dir/note-versions.json" + +python3 "$ci_dir/tasks/eval-diff.py" \ + --rev "${GITHUB_SHA:-$(git rev-parse HEAD)}" \ + --jobs-out "$maps_dir/eval-jobs.jsonl" \ + --map-out "$maps_dir/eval-map.json" \ + --md-out "$maps_dir/rebuild-diff.md" \ + --summary-out "$maps_dir/diff-summary.json" \ + --base-map-out "$maps_dir/base-map.json" \ + --note-versions "$maps_dir/note-versions.json" \ + --priors-out "$maps_dir/note-priors.json" \ + --base-rev "${candidates[@]}" + +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + cat "$maps_dir/rebuild-diff.md" >>"$GITHUB_STEP_SUMMARY" +fi + +NOTE_PRIORS=$(cat "$maps_dir/note-priors.json") nix eval --json --impure \ + ".#legacyPackages.$sys.updateNotes.fired" \ + --apply 'f: f (builtins.fromJSON (builtins.getEnv "NOTE_PRIORS"))' \ + --option accept-flake-config true >"$maps_dir/update-notes.json" || + echo '{}' >"$maps_dir/update-notes.json" + +python3 "$ci_dir/report/eval-fragment.py" \ + --summary "$maps_dir/diff-summary.json" \ + --markdown "$maps_dir/rebuild-diff.md" \ + --notes "$maps_dir/update-notes.json" \ + --out "$fragments_dir/eval.json" diff --git a/scripts/ci/tests/test_comment.py b/scripts/ci/tests/test_comment.py new file mode 100644 index 00000000..a544cf4a --- /dev/null +++ b/scripts/ci/tests/test_comment.py @@ -0,0 +1,70 @@ +import sys +import unittest +from pathlib import Path + +CI = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(CI)) + +from comment import command_from_body, prepare # noqa: E402 +from request import RequestError # noqa: E402 + + +def event(body="@wasinix build python --from-pr"): + return { + "action": "created", + "repository": {"full_name": "wasix-org/wasmer"}, + "issue": {"number": 42, "pull_request": {"url": "unused"}}, + "comment": { + "id": 1234, + "body": body, + "user": {"login": "octocat"}, + }, + } + + +def api(path): + if path.endswith("/permission"): + return {"permission": "write"} + if path.endswith("/pulls/42"): + return { + "state": "open", + "base": {"repo": {"full_name": "wasix-org/wasmer"}}, + "head": {"sha": "A" * 40}, + } + raise AssertionError(path) + + +class CommentTests(unittest.TestCase): + def test_prepares_immutable_dispatch_context(self): + result = prepare(event(), api, "wasix-org") + self.assertEqual(result["command"], "build python --from-pr") + self.assertEqual(result["origin"]["headSha"], "a" * 40) + self.assertEqual(result["origin"]["pullRequest"], 42) + self.assertEqual(result["concurrency"], "command-wasix-org-wasmer-pr-42") + + def test_requires_write_permission(self): + def read_api(path): + if path.endswith("/permission"): + return {"permission": "read"} + return api(path) + + with self.assertRaisesRegex(RequestError, "needs write permission"): + prepare(event(), read_api, "wasix-org") + + def test_rejects_local_execution(self): + with self.assertRaisesRegex(RequestError, "cannot use --local"): + command_from_body("@wasinix build core --local") + + def test_rejects_multiline_command(self): + with self.assertRaisesRegex(RequestError, "one line"): + command_from_body("@wasinix build core\n--with wasmer=version:7") + + def test_rejects_issue_comment(self): + value = event() + del value["issue"]["pull_request"] + with self.assertRaisesRegex(RequestError, "not on a pull request"): + prepare(value, api, "wasix-org") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/tests/test_compare.py b/scripts/ci/tests/test_compare.py new file mode 100644 index 00000000..3b3c7c9d --- /dev/null +++ b/scripts/ci/tests/test_compare.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +import sys +import tempfile +import unittest +from pathlib import Path + +CI = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(CI)) + +from compare import compare_cases, render # noqa: E402 + + +def request(*jobs): + return { + "selectors": [{"kind": "job", "name": name} for name in jobs], + } + + +def mapping(jobs, errors=None, info=None): + return { + "jobs": jobs, + "errors": errors or {}, + "info": info or {}, + "sets": {}, + } + + +def junit(path, cases): + body = "".join( + f'' + f"{'' if failed else ''}" + for name, failed in cases.items() + ) + path.write_text(f"{body}") + + +class CompareTests(unittest.TestCase): + def test_classifies_regressions_fixes_and_existing_failures(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + base_xml, head_xml = root / "base.xml", root / "head.xml" + junit(base_xml, {"regressed": False, "fixed": True, "existing": True}) + junit(head_xml, {"regressed": True, "fixed": False, "existing": True}) + names = { + name: f"/nix/store/{name}.drv" + for name in ("regressed", "fixed", "existing") + } + result = compare_cases( + request(*names), + mapping(names), + [base_xml], + request(*names), + mapping({**names, "regressed": "/nix/store/new.drv"}), + [head_xml], + ) + self.assertEqual(result["regressions"], ["regressed"]) + self.assertEqual(result["fixes"], ["fixed"]) + self.assertEqual(result["existingFailures"], ["existing"]) + self.assertIn("Build regressions", render(result, "base", "head")) + + def test_reports_new_eval_errors(self): + result = compare_cases( + request("job"), + mapping({"job": "/nix/store/a.drv"}), + [], + request("job"), + mapping({}, {"job": "broken"}), + [], + ) + self.assertEqual(result["newEvalErrors"], ["job"]) + + def test_versions_and_rels_are_rendered(self): + result = compare_cases( + request("changed", "removed"), + mapping( + {"changed": "/nix/store/a.drv", "removed": "/nix/store/r.drv"}, + info={ + "changed": {"version": "1.2.3", "rel": 1}, + "removed": {"version": "4.0", "rel": 2}, + }, + ), + [], + request("changed", "added"), + mapping( + {"changed": "/nix/store/b.drv", "added": "/nix/store/n.drv"}, + info={ + "changed": {"version": "1.2.3", "rel": 2}, + "added": {"version": "5.0", "rel": 1}, + }, + ), + [], + ) + markdown = render(result, "base", "head") + self.assertIn("changed: 1.2.3 -> 1.2.3 r2", markdown) + self.assertIn("added 5.0", markdown) + self.assertIn("removed 4.0 r2", markdown) + + def test_non_signal_derivation_is_not_reported_as_rebuilt(self): + result = compare_cases( + request("validation"), + mapping({"validation": "/nix/store/old.drv"}), + [], + request("validation"), + mapping( + {"validation": "/nix/store/new.drv"}, + info={"validation": {"rebuildSignal": False}}, + ), + [], + ) + self.assertEqual(result["rebuilt"], []) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/tests/test_content_diff.py b/scripts/ci/tests/test_content_diff.py new file mode 100644 index 00000000..a185b1f4 --- /dev/null +++ b/scripts/ci/tests/test_content_diff.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +import importlib.util +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "tasks" / "content-diff.py" +SPEC = importlib.util.spec_from_file_location("content_diff", SCRIPT) +content_diff = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(content_diff) + + +class ContentDiffTests(unittest.TestCase): + def test_checks_are_excluded_by_declared_role(self): + base = { + "jobs": { + "checks.test": "/nix/store/old-check.drv", + "package": "/nix/store/old-package.drv", + } + } + head = { + "jobs": { + "checks.test": "/nix/store/new-check.drv", + "package": "/nix/store/new-package.drv", + }, + "info": { + "checks.test": {"role": "check", "contentDiff": False}, + "package": {"role": "artifact", "contentDiff": True}, + }, + } + included, excluded = content_diff.content_jobs(base, head) + self.assertEqual(included, ["package"]) + self.assertEqual(excluded, 1) + + def test_old_maps_default_checks_to_no_content(self): + base = {"jobs": {"checks.test": "/nix/store/old.drv"}} + head = {"jobs": {"checks.test": "/nix/store/new.drv"}} + self.assertEqual(content_diff.content_jobs(base, head), ([], 1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/tests/test_dispatch.py b/scripts/ci/tests/test_dispatch.py new file mode 100644 index 00000000..77541638 --- /dev/null +++ b/scripts/ci/tests/test_dispatch.py @@ -0,0 +1,101 @@ +import json +import sys +import tempfile +import unittest +from pathlib import Path + +CI = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(CI)) + +from dispatch import request_from_event, validate_request # noqa: E402 +from request import RequestError # noqa: E402 + + +SHA = "a" * 40 + + +def build(case_id=None): + value = { + "schema": 1, + "action": "build", + "source": {"rev": SHA, "patch": None, "workingTree": False}, + "selectors": [{"kind": "set", "name": "core"}], + "overrides": [], + "execution": {"local": False}, + } + if case_id: + value["id"] = case_id + return value + + +class DispatchTests(unittest.TestCase): + def test_accepts_immutable_diff(self): + request = { + "schema": 1, + "action": "diff", + "baseline": "baseline", + "contentDiff": False, + "cases": [build("baseline"), build("candidate-1")], + } + self.assertEqual(validate_request(request), request) + + def test_rejects_local_execution(self): + request = build() + request["execution"]["local"] = True + with self.assertRaisesRegex(RequestError, "local runner"): + validate_request(request) + + def test_rejects_caller_patch(self): + request = build() + request["source"]["patch"] = "b" * 64 + with self.assertRaisesRegex(RequestError, "patches"): + validate_request(request) + + def test_rejects_path_case_id(self): + request = { + "schema": 1, + "action": "diff", + "baseline": "baseline", + "contentDiff": False, + "cases": [build("baseline"), build("../../candidate")], + } + with self.assertRaisesRegex(RequestError, "path component"): + validate_request(request) + + def test_rejects_unknown_fields(self): + request = build() + request["shell"] = "arbitrary" + with self.assertRaisesRegex(RequestError, "unknown build field"): + validate_request(request) + + def test_rejects_path_id_on_single_build(self): + request = build("../build") + with self.assertRaisesRegex(RequestError, "path component"): + validate_request(request) + + def test_rejects_nix_expression_in_spot_keep(self): + request = { + "schema": 1, + "action": "spot", + "source": {"rev": SHA, "patch": None, "workingTree": False}, + "targets": ["exnrefEh.zlib"], + "keep": 'toolchain" ]; builtins.abort "oops"', + "base": SHA, + "overrides": [], + "execution": {"local": False, "dryRun": False}, + } + with self.assertRaisesRegex(RequestError, "comma-separated"): + validate_request(request) + + def test_reads_string_payload(self): + request = build() + with tempfile.TemporaryDirectory() as td: + event = Path(td) / "event.json" + event.write_text( + json.dumps({"client_payload": {"request": json.dumps(request)}}) + ) + self.assertEqual(request_from_event(event), request) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/tests/test_eval_diff.py b/scripts/ci/tests/test_eval_diff.py new file mode 100644 index 00000000..9f5ddc72 --- /dev/null +++ b/scripts/ci/tests/test_eval_diff.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +import importlib.util +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "tasks" / "eval-diff.py" +SPEC = importlib.util.spec_from_file_location("eval_diff", SCRIPT) +eval_diff = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(eval_diff) + + +class EvalDiffTests(unittest.TestCase): + def maps(self): + base = { + "rev": "a" * 40, + "jobs": { + "bar": "/nix/store/old-bar.drv", + "baz": "/nix/store/same-baz.drv", + "gone": "/nix/store/gone.drv", + }, + "errors": {}, + "info": { + "bar": {"version": "1.2.3", "rel": 1}, + "baz": {"version": "1.2.3", "rel": 1}, + "gone": {"version": "4.0.0", "rel": 1}, + }, + } + head = { + "rev": "b" * 40, + "jobs": { + "bar": "/nix/store/new-bar.drv", + "baz": "/nix/store/same-baz.drv", + "foobar": "/nix/store/foobar.drv", + }, + "errors": {}, + "info": { + "bar": {"version": "3.2.1", "rel": 1}, + "baz": {"version": "1.2.3", "rel": 2}, + "foobar": {"version": "1.2.3", "rel": 1}, + }, + } + return base, head + + def test_release_only_change_is_reported_without_rebuild(self): + base, head = self.maps() + diff = eval_diff.diff_of(base, head) + self.assertEqual(diff["rebuilt"], ["bar"]) + self.assertEqual(diff["identityChanged"], ["bar", "baz"]) + self.assertEqual(diff["changed"], ["bar", "baz"]) + + def test_versions_and_rels_render_in_sections(self): + base, head = self.maps() + markdown = eval_diff.render(base, head) + self.assertIn("- `foobar` 1.2.3", markdown) + self.assertIn("- `bar`: 1.2.3 → 3.2.1", markdown) + self.assertIn("- `baz`: 1.2.3 → 1.2.3 r2", markdown) + self.assertIn("- `gone` 4.0.0", markdown) + + def test_old_map_without_info_does_not_change_every_job(self): + base, head = self.maps() + base.pop("info") + diff = eval_diff.diff_of(base, head) + self.assertEqual(diff["changed"], ["bar"]) + self.assertEqual(diff["identityChanged"], []) + + def test_non_signal_derivation_is_not_a_rebuild(self): + base, head = self.maps() + head["info"]["bar"]["rebuildSignal"] = False + diff = eval_diff.diff_of(base, head) + self.assertEqual(diff["rebuilt"], []) + self.assertEqual(diff["identityChanged"], ["bar", "baz"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/tests/test_origin.py b/scripts/ci/tests/test_origin.py new file mode 100644 index 00000000..8c820361 --- /dev/null +++ b/scripts/ci/tests/test_origin.py @@ -0,0 +1,59 @@ +import json +import sys +import tempfile +import unittest +from pathlib import Path + +CI = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(CI)) + +from origin import validate_origin, write_token_target # noqa: E402 +from request import RequestError # noqa: E402 + + +def origin(): + return { + "schema": 1, + "repository": "wasix-org/wasmer", + "pullRequest": 42, + "headSha": "a" * 40, + "commentId": 1234, + "actor": "octocat", + } + + +class OriginTests(unittest.TestCase): + def test_accepts_immutable_origin(self): + self.assertEqual(validate_origin(origin(), "wasix-org"), origin()) + + def test_rejects_other_owner(self): + with self.assertRaisesRegex(RequestError, "owner is not allowed"): + validate_origin(origin(), "other-org") + + def test_rejects_unknown_fields(self): + value = origin() + value["command"] = "build all" + with self.assertRaisesRegex(RequestError, "unknown field"): + validate_origin(value) + + def test_rejects_uppercase_sha(self): + value = origin() + value["headSha"] = "A" * 40 + with self.assertRaisesRegex(RequestError, "lowercase commit SHA"): + validate_origin(value) + + def test_writes_scoped_token_target(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + source = root / "origin.json" + output = root / "github-output" + source.write_text(json.dumps(origin())) + write_token_target(source, output, "wasix-org") + self.assertEqual( + output.read_text(), + "external=true\nowner=wasix-org\nrepository=wasmer\n", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/tests/test_planner.py b/scripts/ci/tests/test_planner.py new file mode 100644 index 00000000..13063460 --- /dev/null +++ b/scripts/ci/tests/test_planner.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +import sys +import unittest +from pathlib import Path + +CI = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(CI)) + +from planner import plan_of # noqa: E402 + + +def build(case_id, *selectors): + return { + "schema": 1, + "id": case_id, + "action": "build", + "selectors": [ + {"kind": "job" if name.startswith("checks.") else "set", "name": name} + for name in selectors + ], + "source": {"rev": "a" * 40}, + "overrides": [], + } + + +class PlannerTests(unittest.TestCase): + def test_python_is_gated_by_core(self): + plan = plan_of(build("case", "python")) + self.assertEqual( + [task["id"] for task in plan["tasks"]], + ["case.treefmt", "case.eval", "case.core", "case.python"], + ) + + def test_exact_jobs_do_not_expand_core(self): + plan = plan_of(build("case", "checks.python")) + self.assertEqual( + [task["id"] for task in plan["tasks"]], + ["case.treefmt", "case.eval", "case.jobs"], + ) + + def test_diff_adds_one_comparison_per_candidate(self): + request = { + "schema": 1, + "action": "diff", + "baseline": "baseline", + "contentDiff": True, + "cases": [build("baseline", "core"), build("candidate-1", "core")], + } + ids = [task["id"] for task in plan_of(request)["tasks"]] + self.assertNotIn("baseline.treefmt", ids) + self.assertIn("candidate-1.treefmt", ids) + self.assertIn("compare.candidate-1", ids) + self.assertIn("content-diff.candidate-1", ids) + treefmt = next( + task + for task in plan_of(request)["tasks"] + if task["id"] == "candidate-1.treefmt" + ) + self.assertTrue(treefmt["blocking"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/tests/test_post.js b/scripts/ci/tests/test_post.js new file mode 100644 index 00000000..a5b23160 --- /dev/null +++ b/scripts/ci/tests/test_post.js @@ -0,0 +1,86 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const test = require("node:test"); + +const post = require("../report/post.js"); + +test("posts command reports to the originating repository", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "wasinix-post-")); + fs.mkdirSync(path.join(root, "report")); + fs.writeFileSync( + path.join(root, "origin.json"), + JSON.stringify({ + schema: 1, + repository: "wasix-org/wasmer", + pullRequest: 42, + headSha: "a".repeat(40), + commentId: 1234, + actor: "octocat", + }), + ); + fs.writeFileSync( + path.join(root, "report", "report.json"), + JSON.stringify({ + title: "required CI passed", + conclusion: "success", + complete: true, + annotations: [{ path: "flake.nix", line: 1, title: "x", message: "y" }], + }), + ); + fs.writeFileSync(path.join(root, "report", "report.md"), "### passed\n"); + + const created = { checks: [], comments: [] }; + const github = { + paginate: async (_method, args) => (args.issue_number ? [] : []), + rest: { + checks: { + listForRef: () => {}, + create: async (args) => created.checks.push(args), + update: async () => assert.fail("unexpected check update"), + }, + pulls: { + get: async () => ({ data: { head: { sha: "a".repeat(40) } } }), + }, + repos: { + listPullRequestsAssociatedWithCommit: async () => ({ data: [] }), + }, + issues: { + listComments: () => {}, + createComment: async (args) => created.comments.push(args), + updateComment: async () => assert.fail("unexpected comment update"), + }, + }, + }; + const context = { + eventName: "workflow_dispatch", + runId: 99, + sha: "b".repeat(40), + payload: {}, + repo: { owner: "wasix-org", repo: "wasinix" }, + }; + + const prior = process.env.CI_RUN_DIR; + process.env.CI_RUN_DIR = root; + try { + await post({ + github, + context, + core: { info: () => {}, warning: () => {} }, + }); + } finally { + if (prior === undefined) delete process.env.CI_RUN_DIR; + else process.env.CI_RUN_DIR = prior; + fs.rmSync(root, { recursive: true, force: true }); + } + + assert.equal(created.checks.length, 1); + assert.equal(created.checks[0].owner, "wasix-org"); + assert.equal(created.checks[0].repo, "wasmer"); + assert.equal(created.checks[0].name, "Wasinix CI"); + assert.deepEqual(created.checks[0].output.annotations, undefined); + assert.equal(created.comments.length, 1); + assert.match(created.comments[0].body, /wasinix-ci-command:1234/); + assert.match(created.comments[0].body, /actions\/runs\/99/); +}); diff --git a/scripts/ci/tests/test_report.py b/scripts/ci/tests/test_report.py new file mode 100644 index 00000000..13a13cee --- /dev/null +++ b/scripts/ci/tests/test_report.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +import sys +import unittest +from pathlib import Path + +REPORT_DIR = Path(__file__).resolve().parents[1] / "report" +sys.path.insert(0, str(REPORT_DIR)) + +from render import render + + +def fragment(task_id, status="success"): + return { + "schema": 1, + "id": task_id, + "label": task_id.title(), + "kind": "test", + "status": status, + "headline": status, + "markdown": "", + "annotations": [], + "data": {}, + } + + +class RenderTests(unittest.TestCase): + def plan(self, content=True): + return { + "tasks": [ + { + "id": "required", + "label": "Required", + "order": 10, + "blocking": True, + "enabled": True, + }, + { + "id": "content", + "label": "Content", + "order": 20, + "blocking": False, + "enabled": content, + }, + ] + } + + def test_optional_pending_does_not_hold_check_open(self): + _, report = render(self.plan(), {"required": fragment("required")}, None) + self.assertTrue(report["complete"]) + self.assertEqual(report["conclusion"], "success") + self.assertEqual(report["tasks"][1]["status"], "pending") + + def test_optional_failure_is_advisory(self): + _, report = render( + self.plan(), + { + "required": fragment("required"), + "content": fragment("content", "neutral"), + }, + None, + ) + self.assertEqual(report["conclusion"], "success") + self.assertIn("advisory failure", report["title"]) + + def test_required_failure_fails_snapshot(self): + _, report = render( + self.plan(content=False), + {"required": fragment("required", "failure")}, + None, + ) + self.assertTrue(report["complete"]) + self.assertEqual(report["conclusion"], "failure") + + def test_failure_is_terminal_even_with_downstream_pending(self): + plan = self.plan(content=False) + plan["tasks"].append( + { + "id": "later", + "label": "Later", + "order": 30, + "blocking": True, + "enabled": True, + } + ) + _, report = render(plan, {"required": fragment("required", "failure")}, None) + self.assertTrue(report["complete"]) + self.assertEqual(report["conclusion"], "failure") + + def test_invalid_fragment_status_fails_closed(self): + fragments = { + "required": fragment("required", "passing-ish"), + } + _, report = render(self.plan(content=False), fragments, None) + self.assertEqual(report["tasks"][0]["status"], "failure") + self.assertEqual(report["conclusion"], "failure") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/tests/test_request.py b/scripts/ci/tests/test_request.py new file mode 100644 index 00000000..3057fabb --- /dev/null +++ b/scripts/ci/tests/test_request.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +CI = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(CI)) + +import request # noqa: E402 + + +class ParseTests(unittest.TestCase): + def test_build_sets_jobs_and_overrides(self): + parsed = request.parse_command( + [ + "build", + "python", + "attr:wasmerPackages.git.webc", + "--at", + "main", + "--with", + f"wasix-libc=rev:{'A' * 40}", + "--with=wasmer=version:7.0.0", + ] + ) + self.assertEqual(parsed["action"], "build") + self.assertEqual(parsed["source"]["ref"], "main") + self.assertEqual(parsed["selectors"][1]["kind"], "job") + self.assertEqual(parsed["overrides"][0]["value"], "a" * 40) + self.assertEqual(parsed["overrides"][1]["kind"], "release") + + def test_diff_contains_complete_build_requests(self): + parsed = request.parse_command( + [ + "diff", + "--content-diff", + "build", + "python", + "--at", + "main", + "--vs", + "build", + "python", + "--from-pr=wasix-org/wasix-libc#12", + "--vs", + "build", + "attr:checks.python", + ] + ) + self.assertTrue(parsed["contentDiff"]) + self.assertEqual(parsed["baseline"], "baseline") + self.assertEqual(len(parsed["cases"]), 3) + self.assertEqual(parsed["cases"][1]["fromPr"], "wasix-org/wasix-libc#12") + + def test_diff_rejects_incomplete_segments(self): + with self.assertRaisesRegex(request.RequestError, "separate complete"): + request.parse_command(["diff", "build", "core", "--vs", "--vs"]) + + def test_revision_must_be_immutable(self): + with self.assertRaisesRegex(request.RequestError, "40-character"): + request.parse_command(["build", "core", "--with", "wasix-libc=rev:PR_HEAD"]) + + def test_spot_uses_explicit_cross_attr(self): + parsed = request.parse_command( + ["spot", "attr:exnrefEh.zlib", "--keep", "toolchain,zlib"] + ) + self.assertEqual(parsed["targets"], ["exnrefEh.zlib"]) + self.assertEqual(parsed["keep"], "toolchain,zlib") + + +class NormalizeTests(unittest.TestCase): + def test_external_pr_becomes_declared_revision_override(self): + parsed = request.parse_command( + ["build", "python", "--from-pr=wasix-org/wasix-libc#12"] + ) + pull = { + "base": {"repo": {"full_name": "wasix-org/wasix-libc"}}, + "head": {"sha": "b" * 40}, + "html_url": "https://github.com/wasix-org/wasix-libc/pull/12", + } + with ( + mock.patch.object(request, "resolve_rev", return_value="a" * 40), + mock.patch.object(request, "resolve_pr", return_value=pull), + mock.patch.dict(os.environ, {"GITHUB_REPOSITORY": "wasix-org/wasinix"}), + ): + normalized = request.normalize_build( + parsed, + Path("."), + {"wasix-org/wasix-libc": ["wasix-libc"]}, + ) + self.assertNotIn("fromPr", normalized) + self.assertEqual(normalized["source"]["rev"], "a" * 40) + self.assertEqual(normalized["overrides"][0]["value"], "b" * 40) + self.assertEqual(normalized["overrides"][0]["target"], "wasix-libc") + + def test_wasinix_pr_selects_the_pr_checkout(self): + parsed = request.parse_command(["build", "core", "--from-pr"]) + pull = { + "base": {"repo": {"full_name": "wasix-org/wasinix"}}, + "head": {"sha": "c" * 40}, + } + with ( + mock.patch.object(request, "resolve_rev", return_value="a" * 40), + mock.patch.object(request, "resolve_pr", return_value=pull), + mock.patch.dict(os.environ, {"GITHUB_REPOSITORY": "wasix-org/wasinix"}), + ): + normalized = request.normalize_build(parsed, Path(".")) + self.assertEqual(normalized["source"]["rev"], "c" * 40) + self.assertEqual(normalized["overrides"], []) + + def test_request_id_is_stable(self): + value = {"schema": 1, "action": "build", "selectors": []} + self.assertEqual(request.request_id(value), request.request_id(dict(value))) + value["requestId"] = "stale" + without_id = {key: item for key, item in value.items() if key != "requestId"} + self.assertEqual(request.request_id(value), request.request_id(without_id)) + + def test_repository_is_discovered_from_ssh_remote(self): + with ( + mock.patch.object( + request, "git", return_value="git@github.com:wasix-org/wasinix.git" + ), + mock.patch.dict(os.environ, {}, clear=True), + ): + self.assertEqual(request.current_repository(Path(".")), "wasix-org/wasinix") + + def test_current_pr_uses_immutable_command_origin(self): + origin = { + "repository": "wasix-org/wasmer", + "pullRequest": 42, + "headSha": "d" * 40, + } + with tempfile.TemporaryDirectory() as td: + path = Path(td) / "origin.json" + path.write_text(json.dumps(origin)) + with mock.patch.dict(os.environ, {"WASINIX_CI_ORIGIN": str(path)}): + pull = request.current_pr() + self.assertEqual(pull["base"]["repo"]["full_name"], "wasix-org/wasmer") + self.assertEqual(pull["head"]["sha"], "d" * 40) + + def test_bare_from_pr_uses_command_origin_as_override(self): + parsed = request.parse_command(["build", "python", "--from-pr"]) + origin = { + "repository": "wasix-org/wasix-libc", + "pullRequest": 42, + "headSha": "d" * 40, + } + with tempfile.TemporaryDirectory() as td: + path = Path(td) / "origin.json" + path.write_text(json.dumps(origin)) + with ( + mock.patch.object(request, "resolve_rev", return_value="a" * 40), + mock.patch.dict( + os.environ, + { + "GITHUB_REPOSITORY": "wasix-org/wasinix", + "WASINIX_CI_ORIGIN": str(path), + }, + ), + ): + normalized = request.normalize_build( + parsed, + Path("."), + {"wasix-org/wasix-libc": ["wasix-libc"]}, + ) + self.assertEqual(normalized["overrides"][0]["target"], "wasix-libc") + self.assertEqual(normalized["overrides"][0]["value"], "d" * 40) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/tests/test_workspace.py b/scripts/ci/tests/test_workspace.py new file mode 100644 index 00000000..b2c929e7 --- /dev/null +++ b/scripts/ci/tests/test_workspace.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +CI = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(CI)) + +import request # noqa: E402 +import workspace # noqa: E402 + + +def git(repo, *args): + return subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + text=True, + capture_output=True, + ).stdout.strip() + + +class WorkspaceTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.repo = Path(self.temp.name) / "repo" + self.repo.mkdir() + git(self.repo, "init", "-q") + git(self.repo, "config", "user.email", "ci@example.invalid") + git(self.repo, "config", "user.name", "CI Test") + (self.repo / "value.txt").write_text("base\n") + git(self.repo, "add", "value.txt") + git(self.repo, "commit", "-qm", "base") + self.rev = git(self.repo, "rev-parse", "HEAD") + + def tearDown(self): + self.temp.cleanup() + + def build_request(self): + return { + "schema": 1, + "action": "build", + "source": {"rev": self.rev, "patch": None, "workingTree": True}, + "selectors": [{"kind": "set", "name": "core"}], + "overrides": [], + } + + def test_materialization_captures_working_tree_without_mutating_it(self): + (self.repo / "value.txt").write_text("candidate\n") + out = Path(self.temp.name) / "out" + workspace.write_materialization(self.repo, self.build_request(), out) + self.assertEqual((self.repo / "value.txt").read_text(), "candidate\n") + self.assertIn("+candidate", (out / "materialization.patch").read_text()) + written = json.loads((out / "request.json").read_text()) + self.assertEqual(len(written["source"]["patch"]), 64) + + def test_untracked_file_is_rejected(self): + (self.repo / "new.txt").write_text("invisible\n") + with self.assertRaisesRegex(request.RequestError, "untracked files"): + workspace.working_patch(self.repo) + + def test_overrides_are_applied_in_target_order(self): + calls = [] + values = [ + {"target": "z", "kind": "release", "value": "1"}, + {"target": "a", "kind": "revision", "value": "b" * 40}, + ] + with mock.patch.object( + workspace, "run", side_effect=lambda cmd, **kw: calls.append(cmd) + ): + workspace.materialize_overrides(self.repo, values) + self.assertIn("--to-rev", calls[0]) + self.assertIn("--to", calls[1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/workspace.py b/scripts/ci/workspace.py new file mode 100644 index 00000000..a5703c32 --- /dev/null +++ b/scripts/ci/workspace.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Create and reproduce isolated materialized CI case worktrees.""" + +import hashlib +import json +import shutil +import subprocess +import sys +import tempfile +from contextlib import contextmanager +from pathlib import Path + +from request import RequestError, request_id + + +def run(cmd, *, cwd=None, input=None): + print(f" $ {' '.join(map(str, cmd))}", file=sys.stderr) + p = subprocess.run( + [str(v) for v in cmd], cwd=cwd, input=input, text=True, capture_output=True + ) + if p.returncode != 0: + raise RequestError( + f"{cmd[0]} exited {p.returncode}:\n{(p.stderr or p.stdout).strip()}" + ) + return p + + +def git(repo, *args, input=None): + return run(["git", "-C", repo, *args], input=input).stdout + + +def working_patch(repo): + untracked = git(repo, "ls-files", "--others", "--exclude-standard").splitlines() + if untracked: + shown = ", ".join(untracked[:5]) + more = f" and {len(untracked) - 5} more" if len(untracked) > 5 else "" + raise RequestError( + f"untracked files are invisible to flake evaluation: {shown}{more}; " + "git add them or remove them" + ) + return git(repo, "diff", "--binary", "HEAD") + + +def apply_patch(repo, patch): + if patch: + git(repo, "apply", "--index", "--binary", "-", input=patch) + + +def materialize_overrides(worktree, overrides): + for override in sorted(overrides, key=lambda value: value["target"]): + option = "--to" if override["kind"] == "release" else "--to-rev" + run( + [ + sys.executable, + "scripts/update.py", + option, + f"{override['target']}={override['value']}", + ], + cwd=worktree, + ) + + +def write_materialization(repo, request, out_dir, worktree_parent=None): + """Materialize one normalized build/spot request and return its manifest.""" + repo = Path(repo).resolve() + out_dir = Path(out_dir).resolve() + out_dir.mkdir(parents=True, exist_ok=True) + source = request["source"] + initial = working_patch(repo) if source.get("workingTree") else "" + + parent = Path(worktree_parent) if worktree_parent else Path(tempfile.mkdtemp()) + owned_parent = worktree_parent is None + worktree = parent / "worktree" + try: + git(repo, "worktree", "add", "--detach", worktree, source["rev"]) + apply_patch(worktree, initial) + materialize_overrides(worktree, request.get("overrides", [])) + patch = git(worktree, "diff", "--binary", source["rev"]) + patch_path = out_dir / "materialization.patch" + patch_path.write_text(patch) + normalized = json.loads(json.dumps(request)) + normalized["source"]["patch"] = hashlib.sha256(patch.encode()).hexdigest() + normalized["requestId"] = request_id(normalized) + (out_dir / "request.json").write_text( + json.dumps(normalized, indent=2, sort_keys=True) + "\n" + ) + manifest = { + "schema": 1, + "requestId": normalized["requestId"], + "sourceRev": source["rev"], + "patch": patch_path.name, + "patchHash": normalized["source"]["patch"], + } + (out_dir / "materialization.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n" + ) + return manifest + finally: + if worktree.exists(): + git(repo, "worktree", "remove", "--force", worktree) + if owned_parent: + shutil.rmtree(parent, ignore_errors=True) + + +def reproduce(repo, request, patch_path): + """Apply a prepared case patch to a clean checkout at its declared rev.""" + repo = Path(repo).resolve() + actual = git(repo, "rev-parse", "HEAD").strip() + expected = request["source"]["rev"] + if actual != expected: + raise RequestError(f"checkout is at {actual}, request expects {expected}") + patch = Path(patch_path).read_text() + digest = hashlib.sha256(patch.encode()).hexdigest() + if digest != request["source"]["patch"]: + raise RequestError("materialization patch hash does not match request") + apply_patch(repo, patch) + + +@contextmanager +def reproduced_worktree(repo, request, patch_path): + repo = Path(repo).resolve() + parent = Path(tempfile.mkdtemp()) + worktree = parent / "worktree" + try: + git(repo, "worktree", "add", "--detach", worktree, request["source"]["rev"]) + reproduce(worktree, request, patch_path) + yield worktree + finally: + if worktree.exists(): + git(repo, "worktree", "remove", "--force", worktree) + shutil.rmtree(parent, ignore_errors=True) diff --git a/scripts/post-report.js b/scripts/post-report.js deleted file mode 100644 index 20c3e75f..00000000 --- a/scripts/post-report.js +++ /dev/null @@ -1,151 +0,0 @@ -// Create the "Per-package status" check run and, on PRs, upsert the sticky -// report comment. Called via actions/github-script from ci.yml (same-repo -// events, where the job token has write perms) and from test-report.yml -// (fork PRs via workflow_run, where the in-job token is read-only). Reads -// the report files from cwd; missing files degrade to a stub so a cancelled -// run still gets a check. -const fs = require("fs"); - -const read = (f) => { - try { - return fs.readFileSync(f, "utf8"); - } catch { - return null; - } -}; - -module.exports = async ({ github, context, core }) => { - const isWfRun = context.eventName === "workflow_run"; - const run = isWfRun ? context.payload.workflow_run : null; - - const headSha = isWfRun - ? run.head_sha - : context.eventName === "pull_request" - ? context.payload.pull_request.head.sha - : context.sha; - // Preliminary mode: called right after the eval, before the multi-hour - // build, so the rebuild count (the build-time predictor) is on the PR - // immediately. Creates the check run in_progress; the final call updates - // it in place via the id file. - const preliminary = process.env.PRELIMINARY === "1"; - const rawConclusion = isWfRun - ? run.conclusion - : (process.env.BUILD_OUTCOME ?? "failure"); - const conclusion = ["success", "failure", "cancelled"].includes(rawConclusion) - ? rawConclusion - : "neutral"; - - const report = JSON.parse(read("report.json") ?? "{}"); - const diff = JSON.parse(read("diff-summary.json") ?? "{}"); - const evalTitle = diff.evalFailed - ? "eval failed" - : diff.baseRev != null - ? `building: ${diff.rebuilt} of ${diff.total} jobs rebuild` - : "building (no base map to diff against)"; - const title = preliminary - ? evalTitle - : (report.title ?? `no report produced (${rawConclusion})`); - const body = - [read("build-report.md"), read("content-diff.md"), read("rebuild-diff.md")] - .filter(Boolean) - .join("\n\n") || "No report artifacts found."; - // check run output.summary caps at 64k - const summary = - body.length > 60000 ? body.slice(0, 60000) + "\n\n(truncated)" : body; - - const { owner, repo } = context.repo; - // failure annotations anchored at the package definitions (meta.position); - // the API takes at most 50 per request - const annotations = (report.annotations ?? []).slice(0, 50).map((a) => ({ - path: a.path, - start_line: a.line, - end_line: a.line, - annotation_level: "failure", - title: a.title, - message: a.message, - })); - const output = { - title, - summary, - ...(annotations.length ? { annotations } : {}), - }; - const priorId = read("check-run-id"); - if (preliminary) { - const created = await github.rest.checks.create({ - owner, - repo, - name: "Per-package status", - head_sha: headSha, - status: "in_progress", - output, - }); - fs.writeFileSync("check-run-id", String(created.data.id)); - } else if (priorId) { - await github.rest.checks.update({ - owner, - repo, - check_run_id: Number(priorId), - status: "completed", - conclusion, - output, - }); - } else { - await github.rest.checks.create({ - owner, - repo, - name: "Per-package status", - head_sha: headSha, - status: "completed", - conclusion, - output, - }); - } - - let issue_number; - if (isWfRun) { - if (run.event !== "pull_request") return; - // workflow_run.pull_requests is empty for fork PRs; find by head sha - let prs = run.pull_requests; - if (!prs.length) { - const res = await github.rest.repos.listPullRequestsAssociatedWithCommit({ - owner, - repo, - commit_sha: headSha, - }); - prs = res.data.filter((pr) => pr.state === "open"); - } - if (!prs.length) { - core.warning(`no PR found for ${headSha}`); - return; - } - issue_number = prs[0].number; - } else { - if (context.eventName !== "pull_request") return; - issue_number = context.payload.pull_request.number; - } - - const marker = ""; - const commentBody = marker + "\n" + summary; - const comments = await github.paginate(github.rest.issues.listComments, { - owner, - repo, - issue_number, - per_page: 100, - }); - const existing = comments.find((c) => c.body.startsWith(marker)); - if (existing) { - await github.rest.issues.updateComment({ - owner, - repo, - comment_id: existing.id, - body: commentBody, - }); - } else { - await github.rest.issues.createComment({ - owner, - repo, - issue_number, - body: commentBody, - }); - } -}; diff --git a/scripts/pr-upsert-comment.sh b/scripts/pr-upsert-comment.sh index 571c159f..48c0b8e1 100644 --- a/scripts/pr-upsert-comment.sh +++ b/scripts/pr-upsert-comment.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Create-or-update the PR's preview comment, keyed by a hidden marker so it # never clobbers another bot's comment (the CI report upserts the same way, -# see post-report.js). Usage: pr-upsert-comment.sh . +# see ci/report/post.js). Usage: pr-upsert-comment.sh . # Env: GH_TOKEN, GITHUB_REPOSITORY. set -euo pipefail diff --git a/scripts/rebuild-diff.sh b/scripts/rebuild-diff.sh deleted file mode 100644 index 8c4d3dd9..00000000 --- a/scripts/rebuild-diff.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# Eval the CI job map (attr -> drvPath), diff it against the base branch's -# published map to surface what this change rebuilds, and emit the update notes. -# Run in CI (build.yml) via `nix run .#scripts.rebuild-diff`, which provides -# python3 and nix-eval-jobs. Informational, so failures fall back to empty -# rather than aborting. -# Reads GHA env: GITHUB_SHA, GITHUB_STEP_SUMMARY, and BASE_REF (set by the -# workflow from the pull_request / merge_group base). -set -uo pipefail - -candidates=() -if [ -n "${BASE_REF:-}" ]; then - # walk back: the newest base commits may not have published yet - git fetch --quiet --depth=30 origin "$BASE_REF" - mapfile -t candidates < <(git rev-list -n 30 FETCH_HEAD) -fi - -# update notes: current versions ride in the map; the base map's copy comes back -# as the `prior` side of each note's predicate -sys=$(nix eval --raw --impure --expr 'builtins.currentSystem') -nix eval --json ".#legacyPackages.$sys.updateNotes.versions" \ - --option accept-flake-config true >note-versions.json || - echo '{}' >note-versions.json - -python3 scripts/eval-diff.py \ - --rev "$GITHUB_SHA" \ - --jobs-out eval-jobs.jsonl \ - --map-out eval-map.json \ - --md-out rebuild-diff.md \ - --summary-out diff-summary.json \ - --base-map-out base-map.json \ - --note-versions note-versions.json \ - --priors-out note-priors.json \ - --base-rev "${candidates[@]}" - -cat rebuild-diff.md >>"$GITHUB_STEP_SUMMARY" - -NOTE_PRIORS=$(cat note-priors.json) nix eval --json --impure \ - ".#legacyPackages.$sys.updateNotes.fired" \ - --apply 'f: f (builtins.fromJSON (builtins.getEnv "NOTE_PRIORS"))' \ - --option accept-flake-config true >update-notes.json || - echo '{}' >update-notes.json diff --git a/scripts/tests/test_update.py b/scripts/tests/test_update.py new file mode 100644 index 00000000..a3671486 --- /dev/null +++ b/scripts/tests/test_update.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 + +import importlib.util +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +SCRIPTS = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPTS)) + +from updater_lib import ( # noqa: E402 + UPDATE_REQUEST_ENV, + UpdateRequest, + nix_update_argv, + update_request, +) + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +driver = load("update_driver", SCRIPTS / "update.py") +libc_update = load( + "libc_update", SCRIPTS.parent / "pkgs" / "toolchain" / "sysroot" / "update.py" +) + + +class RequestTests(unittest.TestCase): + def target(self): + return driver.Target( + "wasix-libc", + "updateScript", + accepts=("release", "revision"), + source={ + "kind": "github", + "owner": "wasix-org", + "repo": "wasix-libc", + }, + ) + + def test_release_request_is_typed(self): + requests = driver.explicit_requests( + [self.target()], "release", {"wasix-libc": "1.2.3"} + ) + self.assertEqual( + requests["wasix-libc"], + { + "schema": 1, + "mode": "release", + "target": "wasix-libc", + "value": "1.2.3", + }, + ) + + def test_revision_sha_uses_declared_repository(self): + rev = "A" * 40 + request = driver.explicit_requests( + [self.target()], "revision", {"wasix-libc": rev} + )["wasix-libc"] + self.assertEqual( + request["source"], + { + "kind": "github", + "owner": "wasix-org", + "repo": "wasix-libc", + "rev": rev.lower(), + }, + ) + + def test_revision_rejects_a_different_repository(self): + with self.assertRaisesRegex(SystemExit, "revision source must be"): + driver.explicit_requests( + [self.target()], + "revision", + {"wasix-libc": f"github:someone/else@{'a' * 40}"}, + ) + + def test_unsupported_target_is_rejected(self): + target = driver.Target("llvm", "updateScript") + with self.assertRaisesRegex(SystemExit, "does not accept release"): + driver.explicit_requests([target], "release", {"llvm": "22.0.0"}) + + def test_request_round_trips_through_environment(self): + value = { + "schema": 1, + "mode": "revision", + "target": "wasix-libc", + "source": { + "kind": "github", + "owner": "wasix-org", + "repo": "wasix-libc", + "rev": "a" * 40, + }, + } + with mock.patch.dict(os.environ, {UPDATE_REQUEST_ENV: json.dumps(value)}): + request = update_request("wasix-libc") + self.assertEqual(request.mode, "revision") + self.assertEqual(request.source["rev"], "a" * 40) + + def test_malformed_environment_request_is_rejected(self): + with mock.patch.dict(os.environ, {UPDATE_REQUEST_ENV: "[]"}): + with self.assertRaisesRegex(SystemExit, "expected an object"): + update_request() + + def test_release_replaces_the_declared_nix_update_channel(self): + request = UpdateRequest("release", "pkg", "1.2.3") + argv = nix_update_argv( + ["nix-update", "--flake", "--version=branch", "--src-only"], request + ) + self.assertEqual( + argv, + ["nix-update", "--flake", "--src-only", "--version=1.2.3"], + ) + + +class WasixLibcRevisionTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.libc = Path(self.temp.name) / "libc.nix" + self.libc.write_text( + """let + version = "1.2.3"; + src = fetchFromGitHub { + owner = "wasix-org"; + repo = "wasix-libc"; + tag = "v${version}"; + hash = "sha256-old-source"; + }; + wasiWitx = fetchFromGitHub { + owner = "WebAssembly"; + repo = "WASI"; + rev = "1111111111111111111111111111111111111111"; + hash = "sha256-old-wasi"; + }; + wasixWitx = fetchFromGitHub { + owner = "wasix-org"; + repo = "wasix-witx"; + rev = "2222222222222222222222222222222222222222"; + hash = "sha256-old-wasix"; + }; +in {} +""" + ) + + def tearDown(self): + self.temp.cleanup() + + def test_revision_materializes_source_and_derived_pins(self): + rev = "a" * 40 + request = UpdateRequest( + "revision", + "wasix-libc", + source={ + "kind": "github", + "owner": "wasix-org", + "repo": "wasix-libc", + "rev": rev, + }, + ) + submodule_revs = { + "tools/wasi-headers/WASI": "b" * 40, + "tools/wasix-headers/WASI": "c" * 40, + } + + def github(path): + submodule = path.split("/contents/", 1)[1].split("?", 1)[0] + self.assertTrue(path.endswith(f"?ref={rev}")) + return {"sha": submodule_revs[submodule]} + + def prefetch(owner, repo, revision): + return f"sha256-new-{repo}-{revision[0]}" + + with ( + mock.patch.object(libc_update, "LIBC", self.libc), + mock.patch.object(libc_update, "gh", side_effect=github), + mock.patch.object(libc_update, "prefetch_github", side_effect=prefetch), + ): + prior, materialized = libc_update.materialize_revision(request) + synced = libc_update.sync_witx(materialized) + + text = self.libc.read_text() + self.assertEqual(prior, "v1.2.3") + self.assertEqual(materialized, rev) + self.assertIn(f'rev = "{rev}";', text) + self.assertIn('hash = "sha256-new-wasix-libc-a";', text) + self.assertIn(f'rev = "{"b" * 40}";', text) + self.assertIn('hash = "sha256-new-WASI-b";', text) + self.assertIn(f'rev = "{"c" * 40}";', text) + self.assertIn('hash = "sha256-new-wasix-witx-c";', text) + self.assertEqual(synced, "WASI bbbbbbbbbbbb, wasix-witx cccccccccccc") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update.py b/scripts/update.py index 13d2fb4d..ac264a02 100755 --- a/scripts/update.py +++ b/scripts/update.py @@ -13,6 +13,8 @@ # Usage (or `nix run .#scripts.update -- ...`): # scripts/update.py # update everything # scripts/update.py --only llvm wasix-libc +# scripts/update.py --to wasix-libc=2026-08-01.1 +# scripts/update.py --to-rev wasix-libc=<40-character-commit-sha> # scripts/update.py --list # show targets, no changes import argparse @@ -24,7 +26,7 @@ from dataclasses import dataclass from pathlib import Path -from updater_lib import REPO, gh, prefetch_url, run # noqa: F401 +from updater_lib import REPO, UPDATE_REQUEST_ENV, gh, prefetch_url, run # noqa: F401 SYSTEM = "x86_64-linux" @@ -40,6 +42,8 @@ class Target: command: tuple = () command_drv_paths: tuple = () file: str = "" # repo-relative pin file, from meta.position + accepts: tuple = () + source: dict | None = None def prune_rels(): @@ -265,6 +269,8 @@ def discovered_targets(): command=tuple(s["command"]), command_drv_paths=tuple(s["commandDrvPaths"]), file=repo_relative(pos.rsplit(":", 1)[0]) if pos else "", + accepts=tuple(s.get("accepts", [])), + source=s.get("source"), ) return list(targets.values()) @@ -307,7 +313,7 @@ def run_retention_hook(name, command): # caller derive it from whether the working tree changed). -def run_update_script(t): +def run_update_script(t, request=None): cmd = list(t.command) # repo-relative script commands run from the checkout; store paths and # bare tool names pass through @@ -322,6 +328,8 @@ def run_update_script(t): env["UPDATE_NIX_ATTR_PATH"] = t.attr if t.file: env["UPDATE_NIX_SOURCE_FILE"] = t.file + if request is not None: + env[UPDATE_REQUEST_ENV] = json.dumps(request, sort_keys=True) print(f" $ {' '.join(t.command)}", file=sys.stderr) p = subprocess.run(cmd, cwd=REPO, env=env, text=True, capture_output=True) sys.stderr.write(p.stderr) @@ -356,7 +364,7 @@ def flake_input_rev(name): return node.get("rev") or node.get("ref") or "" -def update_flake_input(t): +def update_flake_input(t, request=None): before = flake_input_rev(t.input) run(["nix", "flake", "update", t.input], cwd=REPO) after = flake_input_rev(t.input) @@ -365,7 +373,7 @@ def update_flake_input(t): return outcome -def update_crate_pins(t): +def update_crate_pins(t, request=None): # The overlay registry's crates.json is a pin: crate-pins re-enumerates # crates.io for each mintable crate's `versions` constraint, adding new # matching releases and pruning gone ones so the mint tracks upstream. @@ -378,9 +386,85 @@ def update_crate_pins(t): return line.split(":", 1)[1].strip() if line else None +def parse_assignments(values, option): + result = {} + for spec in values or []: + name, sep, value = spec.partition("=") + if not sep or not name or not value: + raise SystemExit(f"{option} expects NAME=VALUE, got {spec!r}") + if name in result: + raise SystemExit(f"{option} repeats target {name!r}") + result[name] = value + return result + + +def explicit_requests(targets, mode, assignments): + by_name = {t.name: t for t in targets} + unknown = set(assignments) - set(by_name) + if unknown: + raise SystemExit(f"unknown target(s): {', '.join(sorted(unknown))}") + + requests = {} + for name, value in assignments.items(): + target = by_name[name] + if mode not in target.accepts: + supported = ", ".join(target.accepts) or "automatic updates only" + raise SystemExit( + f"{name} does not accept {mode} requests (supports: {supported})" + ) + request = {"schema": 1, "mode": mode, "target": name} + if mode == "release": + request["value"] = value + else: + source = target.source or {} + if source.get("kind") != "github": + raise SystemExit(f"{name} has no GitHub revision source") + match = re.fullmatch(r"github:([^/@]+)/([^@]+)@([0-9a-fA-F]{40})", value) + if match: + requested_owner, requested_repo, rev = match.groups() + if (requested_owner.lower(), requested_repo.lower()) != ( + source.get("owner", "").lower(), + source.get("repo", "").lower(), + ): + raise SystemExit( + f"{name} revision source must be " + f"{source['owner']}/{source['repo']}" + ) + owner, repo = source["owner"], source["repo"] + elif re.fullmatch(r"[0-9a-fA-F]{40}", value): + owner, repo, rev = source["owner"], source["repo"], value + else: + raise SystemExit( + f"{name} revision expects a 40-character commit SHA or " + "github:OWNER/REPO@SHA" + ) + request["source"] = { + "kind": "github", + "owner": owner, + "repo": repo, + "rev": rev.lower(), + } + requests[name] = request + return requests + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--only", nargs="*", metavar="NAME") + explicit = ap.add_mutually_exclusive_group() + explicit.add_argument( + "--to", + action="append", + metavar="NAME=VERSION", + help="materialize an exact release using the target's updater", + ) + explicit.add_argument( + "--to-rev", + dest="to_rev", + action="append", + metavar="NAME=SOURCE", + help="materialize an exact source revision without release housekeeping", + ) ap.add_argument("--list", action="store_true") ap.add_argument( "--list-json", @@ -395,6 +479,19 @@ def main(): args = ap.parse_args() targets = discovered_targets() + TARGETS + if args.only is not None and (args.to or args.to_rev): + ap.error("--only cannot be combined with --to or --to-rev") + + request_mode = "release" if args.to else "revision" if args.to_rev else None + assignments = parse_assignments( + args.to or args.to_rev, + "--to" if args.to else "--to-rev", + ) + requests = ( + explicit_requests(targets, request_mode, assignments) if request_mode else {} + ) + if requests: + targets = [t for t in targets if t.name in requests] if args.only: wanted = set(args.only) targets = [t for t in targets if t.name in wanted] @@ -425,10 +522,12 @@ def repo_status(): "crate-pins": update_crate_pins, } - # captured before anything bumps: the `prior` side of the update notes - priors = note_versions() - # and of regen_history, below. Any target can move a served version. - history_priors.update(current_versions()) + revision = request_mode == "revision" + # Revision materialization preserves release identity and does not mutate + # publication state. Release and automatic updates retain the old behavior. + priors = {} if revision else note_versions() + if not revision: + history_priors.update(current_versions()) # One flaky upstream must not abort the rest: isolate each target, collect # failures, and exit non-zero at the end so CI/the workflow notices. @@ -439,7 +538,7 @@ def repo_status(): print(f"==> {t.name}") before = repo_status() try: - outcome = backends[t.backend](t) + outcome = backends[t.backend](t, requests.get(t.name)) except Exception as e: first = str(e).splitlines()[0][:120] if str(e) else "unknown error" print(f" FAILED: {e}") @@ -454,7 +553,7 @@ def repo_status(): # Retain before pruning: prune_rels drops the rel key of any version no # longer served, which is exactly the version retention just brought back. - if any_changed: + if any_changed and not revision: try: retained = regen_history() if retained: @@ -463,19 +562,20 @@ def repo_status(): failures.append("history retention") results.append(("history", f"FAILED: {str(e).splitlines()[0][:120]}")) - try: - pruned = prune_rels() - if pruned: - results.append(("rels", pruned)) - except Exception as e: - failures.append("rels prune") - results.append(("rels", f"FAILED: {str(e).splitlines()[0][:120]}")) + if not revision: + try: + pruned = prune_rels() + if pruned: + results.append(("rels", pruned)) + except Exception as e: + failures.append("rels prune") + results.append(("rels", f"FAILED: {str(e).splitlines()[0][:120]}")) # Package-declared re-sync, last: a hook regenerates a listing derived from # the pins (icu's versions.nix) once history and prune have settled. Each is # isolated like a target, and reads the pins directly, so a hook can repair # a listing even when a stale one breaks the repo eval. - if any_changed: + if any_changed and not revision: for name, command in discovered_hooks(): before = repo_status() try: @@ -490,7 +590,7 @@ def repo_status(): if repo_status() != before: results.append((name, outcome or "re-synced")) - notes = fired_notes(priors) + notes = [] if revision else fired_notes(priors) for n in notes: moved = f" ({n['prior']} -> {n['version']})" if n.get("prior") else "" print(f"\nNOTE: {n['name']}{moved}:\n {n['message']}") diff --git a/scripts/updater_lib.py b/scripts/updater_lib.py index 00ac54cd..d7b7b8a3 100644 --- a/scripts/updater_lib.py +++ b/scripts/updater_lib.py @@ -6,12 +6,86 @@ # the pins being edited are in the working tree. import json +import os import subprocess import sys +from dataclasses import dataclass from pathlib import Path from urllib import request +UPDATE_REQUEST_ENV = "WASINIX_UPDATE_REQUEST" + + +@dataclass(frozen=True) +class UpdateRequest: + mode: str + target: str + value: str = "" + source: dict | None = None + + +def update_request(expected_target=None): + """Read the driver's typed explicit-update request, if there is one.""" + raw = os.environ.get(UPDATE_REQUEST_ENV) + if not raw: + return None + try: + value = json.loads(raw) + except json.JSONDecodeError as e: + raise SystemExit(f"invalid {UPDATE_REQUEST_ENV}: {e}") from e + if not isinstance(value, dict): + raise SystemExit(f"invalid {UPDATE_REQUEST_ENV}: expected an object") + if value.get("schema") != 1 or value.get("mode") not in {"release", "revision"}: + raise SystemExit(f"unsupported {UPDATE_REQUEST_ENV} value") + target = value.get("target") + if not isinstance(target, str) or not target: + raise SystemExit(f"{UPDATE_REQUEST_ENV} has no target") + if expected_target is not None and target != expected_target: + raise SystemExit( + f"{UPDATE_REQUEST_ENV} targets {target!r}, expected {expected_target!r}" + ) + request = UpdateRequest( + mode=value["mode"], + target=target, + value=value.get("value", ""), + source=value.get("source"), + ) + if request.mode == "release" and not request.value: + raise SystemExit(f"{UPDATE_REQUEST_ENV} release request has no value") + if request.mode == "revision" and not isinstance(request.source, dict): + raise SystemExit(f"{UPDATE_REQUEST_ENV} revision request has no source") + return request + + +def nix_update_argv(argv, request): + """Apply a release request to a declared nix-update argv.""" + if request is None: + return list(argv) + if request.mode != "release": + raise SystemExit(f"nix-update cannot materialize a {request.mode} request") + + # The explicit request wins over a package's normal channel selector such + # as --version=branch. Remove it instead of relying on argparse's handling + # of repeated options. + out = [] + i = 0 + while i < len(argv): + arg = argv[i] + if arg == "--version": + i += 1 + if i < len(argv) and not argv[i].startswith("-"): + i += 1 + continue + if arg.startswith("--version="): + i += 1 + continue + out.append(arg) + i += 1 + out.append(f"--version={request.value}") + return out + + def repo_root(): try: out = subprocess.run( @@ -71,12 +145,14 @@ def prefetch_github(owner, repo, rev): return json.loads(out.stdout)["hash"] -def run_nix_update(argv): +def run_nix_update(argv, request=None): """Run the nix-update command the package declared, streaming its output. The driver passes it as our argv, so the package keeps using nix-update-script rather than restating its arguments here.""" if not argv: raise SystemExit("no nix-update command passed") + request = update_request() if request is None else request + argv = nix_update_argv(argv, request) p = subprocess.run(argv, cwd=REPO, text=True, capture_output=True) sys.stderr.write(p.stderr) sys.stdout.write(p.stdout)