Build and Publish Docker Images #1813
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Publish Docker Images | |
| on: | |
| schedule: | |
| - cron: '0 4 * * *' | |
| - cron: '0 16 * * *' | |
| push: | |
| branches: [ "main" ] | |
| workflow_dispatch: | |
| # No workflow-level concurrency: the expensive stages are safe to overlap. | |
| # A manifest is fused only from tags carrying both the commit and the run's | |
| # timestamp (see ci/docker.run_tag), so two runs building simultaneously cannot | |
| # contribute images to each other's manifest. Only the final publish of the | |
| # floating multi-arch tags is serialised -- see the create-manifest job. | |
| env: | |
| DOCKER_IMAGE_NAME: ${{ github.repository }} | |
| DOCKER_REGISTRY: ghcr.io | |
| GITHUB_SHA: ${{ github.sha }} | |
| MAX_RETRIES: 50 | |
| # Workers per platform. Each runs cpu_count() concurrent builds and steals | |
| # from its peers when it runs dry, so this only needs to be in the right | |
| # neighbourhood -- it is not a partition that has to be balanced. | |
| WORKER_COUNT: 4 | |
| # Concurrent builds per runner -- build workers and reconcile alike, so both | |
| # stages present the same load to a runner. Empirical, not architectural: | |
| # these builds wait on the network far more than on a core, so the CPU count | |
| # is only a starting point. Tune against the effective-parallelism figure each | |
| # worker reports in the job summary, and watch disk -- that collides first. | |
| BUILD_SLOTS: 4 | |
| UV_PROJECT: .github/scripts | |
| jobs: | |
| plan: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| date: ${{ steps.timestamp.outputs.date }} | |
| date_time: ${{ steps.timestamp.outputs.date_time }} | |
| source_date_epoch: ${{ steps.timestamp.outputs.source_date_epoch }} | |
| matrix: ${{ steps.discover.outputs.matrix }} | |
| reconcile_matrix: ${{ steps.discover.outputs.reconcile_matrix }} | |
| images: ${{ steps.discover.outputs.images }} | |
| platforms: ${{ steps.discover.outputs.platforms }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: .github/scripts/uv.lock | |
| - name: Generate build timestamp | |
| id: timestamp | |
| run: | | |
| # One timestamp for the whole run, so every image in it agrees. | |
| now=$(date '+%s') | |
| date=$(date -u -d "@$now" '+%Y-%m-%d') | |
| date_time=$(date -u -d "@$now" '+%Y-%m-%d.%H-%M-%S') | |
| # Pin SOURCE_DATE_EPOCH to the start of the month so image digests | |
| # stay reproducible across the runs within it. | |
| year_month=$(date -u -d "@$now" '+%Y-%m') | |
| source_date_epoch=$(date -u -d "${year_month}-01 00:00:00" '+%s') | |
| echo "date=$date" >> $GITHUB_OUTPUT | |
| echo "date_time=$date_time" >> $GITHUB_OUTPUT | |
| echo "source_date_epoch=$source_date_epoch" >> $GITHUB_OUTPUT | |
| # Single source of truth for the work list: deals disjoint task shares to | |
| # the workers and mints the run-scoped mesh secret. | |
| - name: Discover and deal build tasks | |
| id: discover | |
| run: uv run python .github/scripts/discover_tasks.py | |
| env: | |
| PLATFORMS: amd64,arm64 | |
| WORKER_COUNT: ${{ env.WORKER_COUNT }} | |
| MAX_RETRIES: ${{ env.MAX_RETRIES }} | |
| build: | |
| needs: plan | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| # One worker's failure must not cancel its peers: they may already hold | |
| # tasks stolen from it, and reconcile handles whatever is left over. | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} | |
| permissions: | |
| contents: write # publish and read the mesh rendezvous refs | |
| packages: write | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: .github/scripts/uv.lock | |
| - name: Log into Docker Registry ${{ env.DOCKER_REGISTRY }} | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.DOCKER_REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| # Repairs this runner's network before anything depends on it. Hosted | |
| # runners egress from ranges shared by a very large number of tenants, | |
| # and some upstreams blackhole them -- which surfaces as a multi-minute | |
| # stall rather than a refusal. Exports BUILD_PROXY_URL when it succeeds | |
| # and is silent when it does not; see ci/egress.py. | |
| # | |
| # continue-on-error because this is a best-effort improvement to the | |
| # runner, never a prerequisite. Every dependency it has -- GitHub's | |
| # release CDN, Cloudflare's registration API and MASQUE edge -- is | |
| # occasionally down, and on those days the build simply egresses from the | |
| # runner's own address exactly as it does today. The script also traps its | |
| # own exceptions; this covers what the interpreter cannot, such as uv | |
| # failing to resolve or the process being OOM-killed. | |
| # | |
| # It needs no platform input: mihomo runs on the runner, so the script | |
| # reads this machine's architecture rather than the image build target. | |
| # | |
| # timeout-minutes because continue-on-error bounds the *verdict*, not the | |
| # clock: a step that hangs still burns the job's six hours, which is the | |
| # very failure this work exists to remove. Provisioning is two API calls | |
| # and a process launch, so anything past three minutes is stuck. | |
| - name: Set up clean egress | |
| continue-on-error: true | |
| timeout-minutes: 3 | |
| run: uv run python .github/scripts/setup_egress.py | |
| # cloudflared is fetched, digest-checked, and cached by ci/tunnel.py at a | |
| # pinned version -- see that module for why it is not installed here. | |
| - name: Build and push assigned images, stealing when idle | |
| run: uv run python .github/scripts/build_docker_images.py | |
| env: | |
| DATE_STR: ${{ needs.plan.outputs.date }} | |
| DATE_TIME_STR: ${{ needs.plan.outputs.date_time }} | |
| DOCKER_PLATFORM: ${{ matrix.platform }} | |
| SOURCE_DATE_EPOCH: ${{ needs.plan.outputs.source_date_epoch }} | |
| WORKER_ID: ${{ matrix.worker_id }} | |
| WORKER_COUNT: ${{ env.WORKER_COUNT }} | |
| BUILD_SLOTS: ${{ env.BUILD_SLOTS }} | |
| WORKER_TASKS: ${{ toJSON(matrix.tasks) }} | |
| # Optional. A repository secret rather than a job output: GitHub | |
| # scrubs masked values out of outputs entirely, and echoes step env | |
| # values into the log, so an unmasked hand-off is not an option | |
| # either. Absent, workers build only their dealt share. | |
| MESH_SECRET: ${{ secrets.MESH_SECRET }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Runs however the build stage ended. The registry decides what actually got | |
| # built; anything missing is rebuilt here. | |
| # | |
| # One job per architecture, each on a runner of that architecture and with the | |
| # same BUILD_SLOTS parallelism as a build worker. A rebuild is a real build, so | |
| # cross-building here would put every missing arm64 image through binfmt/QEMU | |
| # on an amd64 host -- an emulated rebuild of the slowest images in the | |
| # repository, at the point in the run where there is least time to spare. | |
| reconcile: | |
| needs: [plan, build] | |
| if: always() && needs.plan.result == 'success' | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| # One architecture's reconcile must not cancel the other's: they repair | |
| # disjoint image sets, and each is the only job that can repair its own. | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.plan.outputs.reconcile_matrix) }} | |
| permissions: | |
| contents: write # delete this platform's mesh refs | |
| packages: write | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: .github/scripts/uv.lock | |
| - name: Log into Docker Registry ${{ env.DOCKER_REGISTRY }} | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.DOCKER_REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| # Reconcile performs real builds, so it needs the same repaired egress a | |
| # build worker gets -- and needs it more, being the last chance an image | |
| # has to land. Best-effort and time-bounded here too: see the build job. | |
| - name: Set up clean egress | |
| continue-on-error: true | |
| timeout-minutes: 3 | |
| run: uv run python .github/scripts/setup_egress.py | |
| - name: Verify every expected image landed, rebuild what did not | |
| run: uv run python .github/scripts/reconcile_builds.py | |
| env: | |
| DATE_STR: ${{ needs.plan.outputs.date }} | |
| DATE_TIME_STR: ${{ needs.plan.outputs.date_time }} | |
| SOURCE_DATE_EPOCH: ${{ needs.plan.outputs.source_date_epoch }} | |
| MAX_RETRIES: ${{ env.MAX_RETRIES }} | |
| BUILD_SLOTS: ${{ env.BUILD_SLOTS }} | |
| DOCKER_PLATFORM: ${{ matrix.platform }} | |
| IMAGES: ${{ needs.plan.outputs.images }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| create-manifest: | |
| needs: [plan, reconcile] | |
| runs-on: ubuntu-24.04 | |
| # The only serialised stage. It is the one that moves the floating | |
| # multi-arch tags users actually pull, so two runs must not write them at | |
| # once. cancel-in-progress stays false: cancelling here mid-way would leave | |
| # some images fused and some not, which is the one tear that is visible to | |
| # anyone pulling by name. The job is registry-side and short, so the queue | |
| # it creates is short too. | |
| concurrency: | |
| group: ${{ github.workflow }}-publish | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: .github/scripts/uv.lock | |
| - name: Log into Docker Registry ${{ env.DOCKER_REGISTRY }} | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.DOCKER_REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Create and Push Docker Manifests | |
| run: uv run python .github/scripts/create_docker_manifests.py | |
| env: | |
| DATE_STR: ${{ needs.plan.outputs.date }} | |
| DATE_TIME_STR: ${{ needs.plan.outputs.date_time }} | |
| IMAGES: ${{ needs.plan.outputs.images }} | |
| PLATFORMS: ${{ needs.plan.outputs.platforms }} | |
| MAX_RETRIES: ${{ env.MAX_RETRIES }} | |
| # Fast feedback on the orchestration code itself, independent of any build. | |
| check-scripts: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v6 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: .github/scripts/uv.lock | |
| # Both run from the repository root: UV_PROJECT is a workflow-level | |
| # relative path, so a step that changes directory would resolve it | |
| # against the new working directory and fail to find the project. | |
| - name: Lint | |
| run: uv run ruff check .github/scripts | |
| # The scheduler, mesh, and egress paths model their outcomes as closed | |
| # sums eliminated with `assert_never`. Exhaustiveness is a compile-time | |
| # property or it is nothing, so this step is what gives those types force: | |
| # adding a variant without handling it fails here rather than at 3am on a | |
| # runner. | |
| # The config path is explicit because mypy resolves `[tool.mypy]` against | |
| # the working directory, not against the project it was told to run in -- | |
| # so a bare `uv run mypy` from the repository root finds no settings and | |
| # exits complaining it was given nothing to check. | |
| - name: Typecheck | |
| run: uv run mypy --config-file .github/scripts/pyproject.toml .github/scripts | |
| - name: Test | |
| run: uv run pytest .github/scripts/tests |