ci.yml

name: CI

on:
  push:
    branches: [master]
    paths-ignore:
      - '**/*.md'
  pull_request:
    paths-ignore:
      - '**/*.md'

# Collapse redundant runs per ref. Superseded PR runs are cancelled as soon as a
# new commit lands on the branch, but pushes to master are never cancelled
# mid-run: each master commit must finish its own build and push.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  CST_VERSION: 1.22.1
  DAVE_VERSION: 0.5.0
  # Concurrency within a dave command type (dave >= 0.5.0 --jobs). Builds are
  # I/O- and CPU-heavy so they get modest parallelism; structure tests are
  # cheap (docker create + file assertions) so they fan out wider; integration
  # tests each boot a database server, so keep them narrow to bound peak
  # memory on the 4-vCPU runners.
  DAVE_JOBS_BUILD: '3'
  DAVE_JOBS_ST: '4'
  DAVE_JOBS_TEST: '2'

jobs:
  unit:
    # Fast Python-level tests for the transform hooks. Runs first and gates the
    # image build below, so a logic regression fails in seconds rather than
    # after the multi-arch build. Pin to the interpreter the alpine:3.21 builder
    # ships (python3 3.12) so tests exercise the same behavior as production.
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-python@v7
        with:
          python-version: '3.12'

      - name: Install test dependencies
        run: pip install -r test/requirements-dev.txt

      - name: Run unit tests
        run: python -m pytest test/unit -q

  test:
    needs: unit
    strategy:
      fail-fast: false
      matrix:
        # 5 contexts x 2 native runners = 10 shards. Sharding by context bounds
        # the critical path at the slowest single context on one arch instead
        # of every context serially on one runner, and each shard's dave run
        # is further parallelized within a command type via --jobs.
        context: [postgres, mysql, sqlite, cockroach, duckdb]
        arch: [amd64, arm64]
        include:
          - arch: amd64
            runner: ubuntu-latest
            platform: linux/amd64
            cst_arch: amd64
          - arch: arm64
            runner: ubuntu-24.04-arm
            platform: linux/arm64
            cst_arch: arm64
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 30
    env:
      # Build only this runner's native platform; the buildcache refs are
      # arch-scoped so the two arches never clobber each other's cache (the
      # manifest templates keep the unsuffixed legacy ref as a read fallback).
      PLATFORM: ${{ matrix.platform }}
      CACHE_ARCH: -${{ matrix.arch }}
      # True exactly when this run can read repo secrets: a push (on.push only
      # fires for master) or a PR whose head branch lives in this repository.
      # Fork PRs get no secrets by GitHub's own model, so for them this is
      # false -- no login happens, CACHE_TO_SCOPE is never set, and the build
      # renders no --cache-to at all: fork PRs stay import-only, exactly the
      # behavior they have always had.
      REGISTRY_AUTH: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-node@v7
        with:
          node-version: lts/*

      - name: Install dave
        run: npm install -g dave@"$DAVE_VERSION"

      - name: Install container-structure-test
        run: |
          curl -fsSL -o /tmp/cst \
            "https://github.com/GoogleContainerTools/container-structure-test/releases/download/v${CST_VERSION}/container-structure-test-linux-${{ matrix.cst_arch }}"
          chmod +x /tmp/cst
          sudo mv /tmp/cst /usr/local/bin/container-structure-test

      # With the default docker-container driver, `--load` round-trips every
      # built image through a tarball export/import before it is runnable --
      # pure overhead for images whose next stop is the local structure and
      # integration tests. The docker driver builds straight into the daemon's
      # image store, so the image is immediately runnable, but it only gets
      # full BuildKit feature parity (registry cache-from/cache-to, proper
      # multi-platform handling) when the daemon uses the containerd image
      # store. Flip that on before buildx is configured; the jq merge
      # preserves any daemon.json the runner image already ships.
      - name: Enable containerd image store
        run: |
          sudo mkdir -p /etc/docker
          [ -s /etc/docker/daemon.json ] || echo '{}' | sudo tee /etc/docker/daemon.json > /dev/null
          jq '.features["containerd-snapshotter"] = true' /etc/docker/daemon.json > "$RUNNER_TEMP/daemon.json"
          sudo mv "$RUNNER_TEMP/daemon.json" /etc/docker/daemon.json
          sudo systemctl restart docker

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4
        with:
          driver: docker

      # Master pushes need credentials to publish; same-repo PRs need them
      # only so their cache export below can write to the registry.
      - name: Log in to Docker Hub
        if: env.REGISTRY_AUTH == 'true'
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      # Opt in to importing the PR-scoped cache refs (buildcache-<tag>-<arch>-pr)
      # during every build below, so a re-run attempt of this PR hits the cache
      # its own earlier attempt exported. Import only: the matching EXPORT
      # (CACHE_TO_SCOPE) is deliberately not set here but on the dedicated
      # serialized step after the build -- a mode=max cache export is a heavy
      # registry write, and with 10 shards building at -j 3 the fleet peaks at
      # ~30 simultaneous writers, which trips Docker Hub's 429 rate limiting
      # mid-build (the same arithmetic that forced `dave push` to -j 1). Gated
      # on the same-repo condition above: on fork PRs the guard is false, the
      # variable stays unset, and the manifest's ${CACHE_FROM_SCOPE:+...} guard
      # erases the flag entirely.
      - name: Import PR build cache
        if: env.REGISTRY_AUTH == 'true' && github.event_name == 'pull_request'
        run: echo "CACHE_FROM_SCOPE=pr" >> "$GITHUB_ENV"

      # Last-known-good upstream fingerprints, carried across runs. When a live
      # metadata fetch fails, bin/dataset-checksum falls back to the newest
      # fingerprint recorded here instead of degrading to `unknown`; a single
      # transient upstream blip would otherwise change the checksum baked into
      # the build command, bust the EXTRACT layer cache, and force a full
      # re-download of every dataset from the very upstream that just failed.
      # Keyed per run+attempt because caches are immutable once written, and
      # resolved through the context prefix so any earlier run's store is
      # picked up -- cross-run fallback is the whole point of a last-known-good
      # store. Fingerprints do not vary by arch, so arch stays out of the
      # restore prefix and the two shards of a context read each other's
      # entries; it is in the save key only because keys are reserved
      # first-writer-wins and the two shards of one run would otherwise race
      # to claim the same one.
      - name: Restore checksum fallback store
        uses: actions/cache/restore@v5
        with:
          path: ${{ runner.temp }}/checksum-fallback
          key: checksum-lkg-${{ matrix.context }}-${{ matrix.arch }}-${{ github.run_id }}-${{ github.run_attempt }}
          restore-keys: checksum-lkg-${{ matrix.context }}-

      # Integration-test pass stamps from earlier attempts of this same commit.
      # A stamp is keyed by image ID and holds the dataset list plus the exact
      # bytes of every expected JSON, and is only written after a clean pass,
      # so a hit means this exact image already satisfied these exact
      # expectations. Re-running a failed shard re-imports the registry build
      # cache, so tags untouched by the retry rebuild to a byte-identical image
      # ID, hit their stamp, and skip booting the database container entirely;
      # anything whose image or expectations actually changed misses by
      # construction and is tested for real. Scoped to this SHA on purpose: the
      # stamp contents would keep a broader restore correct on their own, but
      # confining it to the commit keeps the cache small and the reasoning
      # local. The restore-keys prefix (no attempt suffix) is what lets attempt
      # N pick up the stamps attempt N-1 wrote.
      - name: Restore integration test stamps
        uses: actions/cache/restore@v5
        with:
          path: ${{ runner.temp }}/itest-stamps
          key: itest-stamps-${{ matrix.context }}-${{ matrix.arch }}-${{ github.sha }}-${{ github.run_attempt }}
          restore-keys: itest-stamps-${{ matrix.context }}-${{ matrix.arch }}-${{ github.sha }}-

      # Every rendered build/push command embeds $(bin/dataset-checksum ...);
      # uncached, that is one upstream metadata round-trip per tag, serialized
      # ahead of any Docker work. Warm them all here concurrently so the
      # per-tag calls become local cache reads. The cache dir is exported via
      # GITHUB_ENV so every later dave step reads from the same warm cache
      # (the runner context is not available in job-level env). The fallback
      # dir is exported the same way, and must be in the environment before
      # this first fetch runs: this step both consumes it (on failure) and
      # refreshes it (on every clean fetch).
      - name: Warm dataset checksums
        env:
          DATASET_CHECKSUM_CACHE: ${{ runner.temp }}/checksums
          DATASET_CHECKSUM_FALLBACK: ${{ runner.temp }}/checksum-fallback
        run: |
          echo "DATASET_CHECKSUM_CACHE=$DATASET_CHECKSUM_CACHE" >> "$GITHUB_ENV"
          echo "DATASET_CHECKSUM_FALLBACK=$DATASET_CHECKSUM_FALLBACK" >> "$GITHUB_ENV"
          # Both persisted dirs exist even if the shard dies before the step
          # that would first write them, so the always() cache saves below
          # never warn about a missing path.
          mkdir -p "$DATASET_CHECKSUM_FALLBACK" "${{ runner.temp }}/itest-stamps"
          bin/dataset-checksums -c ${{ matrix.context }}

      # The builds themselves must not be what pulls the bases: each context
      # has only two or three distinct FROM images, but ~21 tags per shard
      # each resolve them against Docker Hub independently, and 10 shards
      # doing that from one account exhausts the shared hourly pull quota --
      # once it is gone, no amount of minutes-scale backoff brings it back
      # within the job timeout. So pull each distinct base exactly once here
      # (a handful of pulls per shard, well inside the quota) and let every
      # build resolve FROM from the local image store. Stage aliases (`FROM
      # engine`) are names defined by an earlier `FROM ... AS`, not registry
      # refs, and are skipped.
      - name: Pre-pull base images
        run: |
          aliases="$(awk 'toupper($1)=="FROM" && toupper($3)=="AS" {print $4}' ${{ matrix.context }}/Dockerfile | tr '\n' ' ')"
          images="$(awk 'toupper($1)=="FROM" {print $2}' ${{ matrix.context }}/Dockerfile | sort -u)"
          for img in $images; do
            case " $aliases " in *" $img "*) continue ;; esac
            for attempt in 1 2 3 4 5; do
              if docker pull --platform "$PLATFORM" "$img"; then
                break
              fi
              if [ "$attempt" -eq 5 ]; then
                echo "pull of $img failed after 5 attempts" >&2
                exit 1
              fi
              delay=$((attempt * 60))
              echo "pull of $img attempt $attempt failed; retrying in ${delay}s" >&2
              sleep "$delay"
            done
          done

      # Retried like the push below, and for the same reason: with the bases
      # pre-pulled above the remaining registry traffic is buildcache manifest
      # resolution, which can still catch a transient 429. A second
      # `dave build` finds every already-built layer in the local store, so a
      # retry redoes only the tags a failure actually killed.
      - name: Build images
        run: |
          for attempt in 1 2 3; do
            if dave build -c ${{ matrix.context }} -j "$DAVE_JOBS_BUILD"; then
              exit 0
            fi
            if [ "$attempt" -eq 3 ]; then
              echo "build failed after 3 attempts" >&2
              exit 1
            fi
            delay=$((attempt * 60))
            echo "build attempt $attempt failed; retrying in ${delay}s" >&2
            sleep "$delay"
          done

      # The PR cache export, moved out of the build itself (see the Import
      # step above for the concurrency arithmetic). Re-running `dave build`
      # with CACHE_TO_SCOPE set resolves every layer from the local store --
      # the same full-cache-hit trick the Push step uses -- so this is
      # upload-only, and -j 1 keeps the fleet at ~10 concurrent writers, the
      # level the registry tolerates. continue-on-error because the cache is
      # purely an optimization for the next attempt: a throttled export must
      # never fail a shard whose images built and will be tested regardless.
      - name: Export PR build cache
        if: env.REGISTRY_AUTH == 'true' && github.event_name == 'pull_request'
        continue-on-error: true
        env:
          CACHE_TO_SCOPE: pr
        run: dave build -c ${{ matrix.context }} -j 1

      - name: Test images (structure)
        run: dave structure-test -c ${{ matrix.context }} -j "$DAVE_JOBS_ST"

      - name: Test images (integration)
        env:
          DDS_TEST_CACHE: ${{ runner.temp }}/itest-stamps
        run: dave test -c ${{ matrix.context }} -j "$DAVE_JOBS_TEST"

      # Saved with `always()` so a shard that failed part-way still banks the
      # stamps for the tags that did pass -- that partial credit is exactly
      # what makes the next attempt cheap. The key carries the attempt number
      # because a cache entry cannot be overwritten once written; the restore
      # above finds it by prefix.
      - name: Save integration test stamps
        if: always()
        uses: actions/cache/save@v5
        with:
          path: ${{ runner.temp }}/itest-stamps
          key: itest-stamps-${{ matrix.context }}-${{ matrix.arch }}-${{ github.sha }}-${{ github.run_attempt }}

      # Publish this shard's images natively as `<tag>-<arch>`; the merge job
      # below joins the per-arch pairs into the real multi-arch tags. This
      # `dave push` re-runs the same buildx build the Build step ran, against
      # the same daemon on the same runner: under the docker driver every
      # layer is already in the local image store, so the build resolves as a
      # full cache hit and the step is effectively upload-only. The images
      # being pushed are therefore bit-for-bit the ones the tests above
      # validated.
      - name: Push (native arch)
        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
        env:
          TAG_SUFFIX: -${{ matrix.arch }}
        # -j 1: pushes from all matrix shards run simultaneously, so per-shard
        # concurrency multiplies across 10 jobs. At -j 3 that is ~30 concurrent
        # registry writers, which trips Docker Hub's 429 rate limiting; serial
        # pushes per shard keep the fleet at ~10 and uploads are bandwidth-bound
        # anyway. Raise only alongside a Docker Hub plan with higher limits.
        # Even at ~10 writers a 429 still slips through occasionally, so retry
        # up to three times with a lengthening pause: re-running `dave push`
        # hits the same daemon whose image store already holds every layer, so
        # the rebuild resolves as a full cache hit and the retry is
        # upload-only and idempotent -- blobs that already landed are skipped
        # by digest.
        run: |
          for attempt in 1 2 3; do
            if dave push -c ${{ matrix.context }} -j 1; then
              exit 0
            fi
            if [ "$attempt" -eq 3 ]; then
              echo "push failed after 3 attempts" >&2
              exit 1
            fi
            delay=$((attempt * 60))
            echo "push attempt $attempt failed; retrying in ${delay}s" >&2
            sleep "$delay"
          done

      # The fallback store is refreshed by every clean fetch above, so save it
      # at the very end of the job to bank whatever the run learned. `always()`
      # matters here too: a shard that failed in build or test may still have
      # completed a good metadata fetch, and that fingerprint is worth keeping
      # for the next run.
      - name: Save checksum fallback store
        if: always()
        uses: actions/cache/save@v5
        with:
          path: ${{ runner.temp }}/checksum-fallback
          key: checksum-lkg-${{ matrix.context }}-${{ matrix.arch }}-${{ github.run_id }}-${{ github.run_attempt }}

  # A per-arch push leaves `<tag>-amd64` / `<tag>-arm64` in the registry; this
  # joins each pair into the real multi-arch `<tag>` (pass 1) and then points
  # every alias tag -- retagFrom entries such as `latest` -- at its source
  # (pass 2). Both are `imagetools create` calls: manifest-list writes against
  # images already in the registry, so nothing is pulled, rebuilt or
  # re-uploaded, which is why 15 minutes is generous.
  merge:
    needs: test
    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v7

      - name: Log in to Docker Hub
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      # Done-stamps persisted across re-run attempts of this commit, same idea
      # as the integration-test stamps: `imagetools create` re-reads its
      # sources and re-writes the target even for an already-merged tag, so
      # without stamps a rate-limited merge can never converge -- every retry
      # redoes all ~110 tags and dies at the same quota depth. With them, each
      # retry only pays for the tags the previous attempt did not land.
      - name: Restore merge stamps
        uses: actions/cache/restore@v5
        with:
          path: ${{ runner.temp }}/merge-stamps
          key: merge-stamps-${{ github.sha }}-${{ github.run_attempt }}
          restore-keys: merge-stamps-${{ github.sha }}-

      - name: Merge per-arch manifests
        env:
          MERGE_STAMP_DIR: ${{ runner.temp }}/merge-stamps
        run: bin/merge-manifests

      - name: Save merge stamps
        if: always()
        uses: actions/cache/save@v5
        with:
          path: ${{ runner.temp }}/merge-stamps
          key: merge-stamps-${{ github.sha }}-${{ github.run_attempt }}