fix(ci): Repair three defects found by the first real run #1794
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Publish Docker Images | |
| on: | |
| schedule: | |
| - cron: '0 4 * * *' | |
| - cron: '0 16 * * *' | |
| push: | |
| branches: [ "main" ] | |
| workflow_dispatch: | |
| # Serialise every run of this workflow. A run publishes 31 images across 8 | |
| # workers and only then fuses the multi-arch manifests, so two overlapping runs | |
| # would race on the floating tags and leave `:image` and `:image.latest` pointing | |
| # at different commits. | |
| # | |
| # cancel-in-progress is deliberately false. Cancelling mid-run is the one | |
| # outcome that tears a publish: some images would carry the new commit and some | |
| # the old, with no manifest fusing them and no reconcile stage to repair it, | |
| # because run-level cancellation skips `if: always()` jobs too. Queueing costs | |
| # latency instead, and GitHub bounds that for us -- only one run may be pending | |
| # per group, so a third push supersedes the second *before it starts*, where | |
| # cancelling is harmless. | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: false | |
| env: | |
| DOCKER_IMAGE_NAME: ${{ github.repository }} | |
| DOCKER_REGISTRY: ghcr.io | |
| GITHUB_SHA: ${{ github.sha }} | |
| MAX_RETRIES: 50 | |
| # Workers per platform. Each runs cpu_count() concurrent builds and steals | |
| # from its peers when it runs dry, so this only needs to be in the right | |
| # neighbourhood -- it is not a partition that has to be balanced. | |
| WORKER_COUNT: 4 | |
| UV_PROJECT: .github/scripts | |
| jobs: | |
| plan: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| date: ${{ steps.timestamp.outputs.date }} | |
| date_time: ${{ steps.timestamp.outputs.date_time }} | |
| source_date_epoch: ${{ steps.timestamp.outputs.source_date_epoch }} | |
| matrix: ${{ steps.discover.outputs.matrix }} | |
| images: ${{ steps.discover.outputs.images }} | |
| platforms: ${{ steps.discover.outputs.platforms }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: .github/scripts/uv.lock | |
| - name: Generate build timestamp | |
| id: timestamp | |
| run: | | |
| # One timestamp for the whole run, so every image in it agrees. | |
| now=$(date '+%s') | |
| date=$(date -u -d "@$now" '+%Y-%m-%d') | |
| date_time=$(date -u -d "@$now" '+%Y-%m-%d.%H-%M-%S') | |
| # Pin SOURCE_DATE_EPOCH to the start of the month so image digests | |
| # stay reproducible across the runs within it. | |
| year_month=$(date -u -d "@$now" '+%Y-%m') | |
| source_date_epoch=$(date -u -d "${year_month}-01 00:00:00" '+%s') | |
| echo "date=$date" >> $GITHUB_OUTPUT | |
| echo "date_time=$date_time" >> $GITHUB_OUTPUT | |
| echo "source_date_epoch=$source_date_epoch" >> $GITHUB_OUTPUT | |
| # Single source of truth for the work list: deals disjoint task shares to | |
| # the workers and mints the run-scoped mesh secret. | |
| - name: Discover and deal build tasks | |
| id: discover | |
| run: uv run python .github/scripts/discover_tasks.py | |
| env: | |
| PLATFORMS: amd64,arm64 | |
| WORKER_COUNT: ${{ env.WORKER_COUNT }} | |
| MAX_RETRIES: ${{ env.MAX_RETRIES }} | |
| build: | |
| needs: plan | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| # One worker's failure must not cancel its peers: they may already hold | |
| # tasks stolen from it, and reconcile handles whatever is left over. | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} | |
| permissions: | |
| contents: write # publish and read the mesh rendezvous refs | |
| packages: write | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: .github/scripts/uv.lock | |
| - name: Log into Docker Registry ${{ env.DOCKER_REGISTRY }} | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.DOCKER_REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| # cloudflared is fetched, digest-checked, and cached by ci/tunnel.py at a | |
| # pinned version -- see that module for why it is not installed here. | |
| - name: Build and push assigned images, stealing when idle | |
| run: uv run python .github/scripts/build_docker_images.py | |
| env: | |
| DATE_STR: ${{ needs.plan.outputs.date }} | |
| DATE_TIME_STR: ${{ needs.plan.outputs.date_time }} | |
| DOCKER_PLATFORM: ${{ matrix.platform }} | |
| SOURCE_DATE_EPOCH: ${{ needs.plan.outputs.source_date_epoch }} | |
| WORKER_ID: ${{ matrix.worker_id }} | |
| WORKER_COUNT: ${{ env.WORKER_COUNT }} | |
| WORKER_TASKS: ${{ toJSON(matrix.tasks) }} | |
| # Optional. A repository secret rather than a job output: GitHub | |
| # scrubs masked values out of outputs entirely, and echoes step env | |
| # values into the log, so an unmasked hand-off is not an option | |
| # either. Absent, workers build only their dealt share. | |
| MESH_SECRET: ${{ secrets.MESH_SECRET }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Runs however the build stage ended. The registry decides what actually got | |
| # built; anything missing is rebuilt here. | |
| reconcile: | |
| needs: [plan, build] | |
| if: always() && needs.plan.result == 'success' | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: write # delete the run's mesh refs | |
| packages: write | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: .github/scripts/uv.lock | |
| - name: Log into Docker Registry ${{ env.DOCKER_REGISTRY }} | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.DOCKER_REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Verify every expected image landed, rebuild what did not | |
| run: uv run python .github/scripts/reconcile_builds.py | |
| env: | |
| DATE_STR: ${{ needs.plan.outputs.date }} | |
| DATE_TIME_STR: ${{ needs.plan.outputs.date_time }} | |
| SOURCE_DATE_EPOCH: ${{ needs.plan.outputs.source_date_epoch }} | |
| MAX_RETRIES: ${{ env.MAX_RETRIES }} | |
| IMAGES: ${{ needs.plan.outputs.images }} | |
| PLATFORMS: ${{ needs.plan.outputs.platforms }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| create-manifest: | |
| needs: [plan, reconcile] | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: .github/scripts/uv.lock | |
| - name: Log into Docker Registry ${{ env.DOCKER_REGISTRY }} | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.DOCKER_REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Create and Push Docker Manifests | |
| run: uv run python .github/scripts/create_docker_manifests.py | |
| env: | |
| DATE_STR: ${{ needs.plan.outputs.date }} | |
| DATE_TIME_STR: ${{ needs.plan.outputs.date_time }} | |
| IMAGES: ${{ needs.plan.outputs.images }} | |
| PLATFORMS: ${{ needs.plan.outputs.platforms }} | |
| MAX_RETRIES: ${{ env.MAX_RETRIES }} | |
| # Fast feedback on the orchestration code itself, independent of any build. | |
| check-scripts: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: .github/scripts/uv.lock | |
| # Both run from the repository root: UV_PROJECT is a workflow-level | |
| # relative path, so a step that changes directory would resolve it | |
| # against the new working directory and fail to find the project. | |
| - name: Lint | |
| run: uv run ruff check .github/scripts | |
| - name: Test | |
| run: uv run pytest .github/scripts/tests |