Upgrade pagila to current upstream with pgvector support #38

Workflow file for this run

	name: CI

	on:
	push:
	branches: [master]
	paths-ignore:
	- '*/.md'
	pull_request:
	paths-ignore:
	- '*/.md'

	# Collapse redundant runs per ref. Superseded PR runs are cancelled as soon as a
	# new commit lands on the branch, but pushes to master are never cancelled
	# mid-run: each master commit must finish its own build and push.
	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	env:
	CST_VERSION: 1.22.1
	DAVE_VERSION: 0.5.0
	# Concurrency within a dave command type (dave >= 0.5.0 --jobs). Builds are
	# I/O- and CPU-heavy so they get modest parallelism; structure tests are
	# cheap (docker create + file assertions) so they fan out wider; integration
	# tests each boot a database server, so keep them narrow to bound peak
	# memory on the 4-vCPU runners.
	DAVE_JOBS_BUILD: '3'
	DAVE_JOBS_ST: '4'
	DAVE_JOBS_TEST: '2'

	jobs:
	unit:
	# Fast Python-level tests for the transform hooks. Runs first and gates the
	# image build below, so a logic regression fails in seconds rather than
	# after the multi-arch build. Pin to the interpreter the alpine:3.21 builder
	# ships (python3 3.12) so tests exercise the same behavior as production.
	runs-on: ubuntu-latest
	timeout-minutes: 10
	steps:
	- uses: actions/checkout@v7

	- uses: actions/setup-python@v7
	with:
	python-version: '3.12'

	- name: Install test dependencies
	run: pip install -r test/requirements-dev.txt

	- name: Run unit tests
	run: python -m pytest test/unit -q

	test:
	needs: unit
	strategy:
	fail-fast: false
	matrix:
	# 5 contexts x 2 native runners = 10 shards. Sharding by context bounds
	# the critical path at the slowest single context on one arch instead
	# of every context serially on one runner, and each shard's dave run
	# is further parallelized within a command type via --jobs.
	context: [postgres, mysql, sqlite, cockroach, duckdb]
	arch: [amd64, arm64]
	include:
	- arch: amd64
	runner: ubuntu-latest
	platform: linux/amd64
	cst_arch: amd64
	- arch: arm64
	runner: ubuntu-24.04-arm
	platform: linux/arm64
	cst_arch: arm64
	runs-on: ${{ matrix.runner }}
	timeout-minutes: 30
	env:
	# Build only this runner's native platform; the buildcache refs are
	# arch-scoped so the two arches never clobber each other's cache (the
	# manifest templates keep the unsuffixed legacy ref as a read fallback).
	PLATFORM: ${{ matrix.platform }}
	CACHE_ARCH: -${{ matrix.arch }}
	# True exactly when this run can read repo secrets: a push (on.push only
	# fires for master) or a PR whose head branch lives in this repository.
	# Fork PRs get no secrets by GitHub's own model, so for them this is
	# false -- no login happens, CACHE_TO_SCOPE is never set, and the build
	# renders no --cache-to at all: fork PRs stay import-only, exactly the
	# behavior they have always had.
	REGISTRY_AUTH: ${{ github.event_name == 'push' \|\| github.event.pull_request.head.repo.full_name == github.repository }}
	steps:
	- uses: actions/checkout@v7

	- uses: actions/setup-node@v7
	with:
	node-version: lts/*

	- name: Install dave
	run: npm install -g dave@"$DAVE_VERSION"

	- name: Install container-structure-test
	run: \|
	curl -fsSL -o /tmp/cst \
	"https://github.com/GoogleContainerTools/container-structure-test/releases/download/v${CST_VERSION}/container-structure-test-linux-${{ matrix.cst_arch }}"
	chmod +x /tmp/cst
	sudo mv /tmp/cst /usr/local/bin/container-structure-test

	# With the default docker-container driver, `--load` round-trips every
	# built image through a tarball export/import before it is runnable --
	# pure overhead for images whose next stop is the local structure and
	# integration tests. The docker driver builds straight into the daemon's
	# image store, so the image is immediately runnable, but it only gets
	# full BuildKit feature parity (registry cache-from/cache-to, proper
	# multi-platform handling) when the daemon uses the containerd image
	# store. Flip that on before buildx is configured; the jq merge
	# preserves any daemon.json the runner image already ships.
	- name: Enable containerd image store
	run: \|
	sudo mkdir -p /etc/docker
	[ -s /etc/docker/daemon.json ] \|\| echo '{}' \| sudo tee /etc/docker/daemon.json > /dev/null
	jq '.features["containerd-snapshotter"] = true' /etc/docker/daemon.json > "$RUNNER_TEMP/daemon.json"
	sudo mv "$RUNNER_TEMP/daemon.json" /etc/docker/daemon.json
	sudo systemctl restart docker

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v4
	with:
	driver: docker

	# Master pushes need credentials to publish; same-repo PRs need them
	# only so their cache export below can write to the registry.
	- name: Log in to Docker Hub
	if: env.REGISTRY_AUTH == 'true'
	uses: docker/login-action@v4
	with:
	username: ${{ secrets.DOCKERHUB_USERNAME }}
	password: ${{ secrets.DOCKERHUB_TOKEN }}

	# Opt in to importing the PR-scoped cache refs (buildcache-<tag>-<arch>-pr)
	# during every build below, so a re-run attempt of this PR hits the cache
	# its own earlier attempt exported. Import only: the matching EXPORT
	# (CACHE_TO_SCOPE) is deliberately not set here but on the dedicated
	# serialized step after the build -- a mode=max cache export is a heavy
	# registry write, and with 10 shards building at -j 3 the fleet peaks at
	# ~30 simultaneous writers, which trips Docker Hub's 429 rate limiting
	# mid-build (the same arithmetic that forced `dave push` to -j 1). Gated
	# on the same-repo condition above: on fork PRs the guard is false, the
	# variable stays unset, and the manifest's ${CACHE_FROM_SCOPE:+...} guard
	# erases the flag entirely.
	- name: Import PR build cache
	if: env.REGISTRY_AUTH == 'true' && github.event_name == 'pull_request'
	run: echo "CACHE_FROM_SCOPE=pr" >> "$GITHUB_ENV"

	# Last-known-good upstream fingerprints, carried across runs. When a live
	# metadata fetch fails, bin/dataset-checksum falls back to the newest
	# fingerprint recorded here instead of degrading to `unknown`; a single
	# transient upstream blip would otherwise change the checksum baked into
	# the build command, bust the EXTRACT layer cache, and force a full
	# re-download of every dataset from the very upstream that just failed.
	# Keyed per run+attempt because caches are immutable once written, and
	# resolved through the context prefix so any earlier run's store is
	# picked up -- cross-run fallback is the whole point of a last-known-good
	# store. Fingerprints do not vary by arch, so arch stays out of the
	# restore prefix and the two shards of a context read each other's
	# entries; it is in the save key only because keys are reserved
	# first-writer-wins and the two shards of one run would otherwise race
	# to claim the same one.
	- name: Restore checksum fallback store
	uses: actions/cache/restore@v5
	with:
	path: ${{ runner.temp }}/checksum-fallback
	key: checksum-lkg-${{ matrix.context }}-${{ matrix.arch }}-${{ github.run_id }}-${{ github.run_attempt }}
	restore-keys: checksum-lkg-${{ matrix.context }}-

	# Integration-test pass stamps from earlier attempts of this same commit.
	# A stamp is keyed by image ID and holds the dataset list plus the exact
	# bytes of every expected JSON, and is only written after a clean pass,
	# so a hit means this exact image already satisfied these exact
	# expectations. Re-running a failed shard re-imports the registry build
	# cache, so tags untouched by the retry rebuild to a byte-identical image
	# ID, hit their stamp, and skip booting the database container entirely;
	# anything whose image or expectations actually changed misses by
	# construction and is tested for real. Scoped to this SHA on purpose: the
	# stamp contents would keep a broader restore correct on their own, but
	# confining it to the commit keeps the cache small and the reasoning
	# local. The restore-keys prefix (no attempt suffix) is what lets attempt
	# N pick up the stamps attempt N-1 wrote.
	- name: Restore integration test stamps
	uses: actions/cache/restore@v5
	with:
	path: ${{ runner.temp }}/itest-stamps
	key: itest-stamps-${{ matrix.context }}-${{ matrix.arch }}-${{ github.sha }}-${{ github.run_attempt }}
	restore-keys: itest-stamps-${{ matrix.context }}-${{ matrix.arch }}-${{ github.sha }}-

	# Every rendered build/push command embeds $(bin/dataset-checksum ...);
	# uncached, that is one upstream metadata round-trip per tag, serialized
	# ahead of any Docker work. Warm them all here concurrently so the
	# per-tag calls become local cache reads. The cache dir is exported via
	# GITHUB_ENV so every later dave step reads from the same warm cache
	# (the runner context is not available in job-level env). The fallback
	# dir is exported the same way, and must be in the environment before
	# this first fetch runs: this step both consumes it (on failure) and
	# refreshes it (on every clean fetch).
	- name: Warm dataset checksums
	env:
	DATASET_CHECKSUM_CACHE: ${{ runner.temp }}/checksums
	DATASET_CHECKSUM_FALLBACK: ${{ runner.temp }}/checksum-fallback
	run: \|
	echo "DATASET_CHECKSUM_CACHE=$DATASET_CHECKSUM_CACHE" >> "$GITHUB_ENV"
	echo "DATASET_CHECKSUM_FALLBACK=$DATASET_CHECKSUM_FALLBACK" >> "$GITHUB_ENV"
	# Both persisted dirs exist even if the shard dies before the step
	# that would first write them, so the always() cache saves below
	# never warn about a missing path.
	mkdir -p "$DATASET_CHECKSUM_FALLBACK" "${{ runner.temp }}/itest-stamps"
	bin/dataset-checksums -c ${{ matrix.context }}

	# The builds themselves must not be what pulls the bases: each context
	# has only two or three distinct FROM images, but ~21 tags per shard
	# each resolve them against Docker Hub independently, and 10 shards
	# doing that from one account exhausts the shared hourly pull quota --
	# once it is gone, no amount of minutes-scale backoff brings it back
	# within the job timeout. So pull each distinct base exactly once here
	# (a handful of pulls per shard, well inside the quota) and let every
	# build resolve FROM from the local image store. Stage aliases (`FROM
	# engine`) are names defined by an earlier `FROM ... AS`, not registry
	# refs, and are skipped.
	- name: Pre-pull base images
	run: \|
	aliases="$(awk 'toupper($1)=="FROM" && toupper($3)=="AS" {print $4}' ${{ matrix.context }}/Dockerfile \| tr '\n' ' ')"
	images="$(awk 'toupper($1)=="FROM" {print $2}' ${{ matrix.context }}/Dockerfile \| sort -u)"
	for img in $images; do
	case " $aliases " in " $img ") continue ;; esac
	for attempt in 1 2 3 4 5; do
	if docker pull --platform "$PLATFORM" "$img"; then
	break
	fi
	if [ "$attempt" -eq 5 ]; then
	echo "pull of $img failed after 5 attempts" >&2
	exit 1
	fi
	delay=$((attempt * 60))
	echo "pull of $img attempt $attempt failed; retrying in ${delay}s" >&2
	sleep "$delay"
	done
	done

	# Retried like the push below, and for the same reason: with the bases
	# pre-pulled above the remaining registry traffic is buildcache manifest
	# resolution, which can still catch a transient 429. A second
	# `dave build` finds every already-built layer in the local store, so a
	# retry redoes only the tags a failure actually killed.
	- name: Build images
	run: \|
	for attempt in 1 2 3; do
	if dave build -c ${{ matrix.context }} -j "$DAVE_JOBS_BUILD"; then
	exit 0
	fi
	if [ "$attempt" -eq 3 ]; then
	echo "build failed after 3 attempts" >&2
	exit 1
	fi
	delay=$((attempt * 60))
	echo "build attempt $attempt failed; retrying in ${delay}s" >&2
	sleep "$delay"
	done

	# The PR cache export, moved out of the build itself (see the Import
	# step above for the concurrency arithmetic). Re-running `dave build`
	# with CACHE_TO_SCOPE set resolves every layer from the local store --
	# the same full-cache-hit trick the Push step uses -- so this is
	# upload-only, and -j 1 keeps the fleet at ~10 concurrent writers, the
	# level the registry tolerates. continue-on-error because the cache is
	# purely an optimization for the next attempt: a throttled export must
	# never fail a shard whose images built and will be tested regardless.
	- name: Export PR build cache
	if: env.REGISTRY_AUTH == 'true' && github.event_name == 'pull_request'
	continue-on-error: true
	env:
	CACHE_TO_SCOPE: pr
	run: dave build -c ${{ matrix.context }} -j 1

	- name: Test images (structure)
	run: dave structure-test -c ${{ matrix.context }} -j "$DAVE_JOBS_ST"

	- name: Test images (integration)
	env:
	DDS_TEST_CACHE: ${{ runner.temp }}/itest-stamps
	run: dave test -c ${{ matrix.context }} -j "$DAVE_JOBS_TEST"

	# Saved with `always()` so a shard that failed part-way still banks the
	# stamps for the tags that did pass -- that partial credit is exactly
	# what makes the next attempt cheap. The key carries the attempt number
	# because a cache entry cannot be overwritten once written; the restore
	# above finds it by prefix.
	- name: Save integration test stamps
	if: always()
	uses: actions/cache/save@v5
	with:
	path: ${{ runner.temp }}/itest-stamps
	key: itest-stamps-${{ matrix.context }}-${{ matrix.arch }}-${{ github.sha }}-${{ github.run_attempt }}

	# Publish this shard's images natively as `<tag>-<arch>`; the merge job
	# below joins the per-arch pairs into the real multi-arch tags. This
	# `dave push` re-runs the same buildx build the Build step ran, against
	# the same daemon on the same runner: under the docker driver every
	# layer is already in the local image store, so the build resolves as a
	# full cache hit and the step is effectively upload-only. The images
	# being pushed are therefore bit-for-bit the ones the tests above
	# validated.
	- name: Push (native arch)
	if: github.event_name == 'push' && github.ref == 'refs/heads/master'
	env:
	TAG_SUFFIX: -${{ matrix.arch }}
	# -j 1: pushes from all matrix shards run simultaneously, so per-shard
	# concurrency multiplies across 10 jobs. At -j 3 that is ~30 concurrent
	# registry writers, which trips Docker Hub's 429 rate limiting; serial
	# pushes per shard keep the fleet at ~10 and uploads are bandwidth-bound
	# anyway. Raise only alongside a Docker Hub plan with higher limits.
	# Even at ~10 writers a 429 still slips through occasionally, so retry
	# up to three times with a lengthening pause: re-running `dave push`
	# hits the same daemon whose image store already holds every layer, so
	# the rebuild resolves as a full cache hit and the retry is
	# upload-only and idempotent -- blobs that already landed are skipped
	# by digest.
	run: \|
	for attempt in 1 2 3; do
	if dave push -c ${{ matrix.context }} -j 1; then
	exit 0
	fi
	if [ "$attempt" -eq 3 ]; then
	echo "push failed after 3 attempts" >&2
	exit 1
	fi
	delay=$((attempt * 60))
	echo "push attempt $attempt failed; retrying in ${delay}s" >&2
	sleep "$delay"
	done

	# The fallback store is refreshed by every clean fetch above, so save it
	# at the very end of the job to bank whatever the run learned. `always()`
	# matters here too: a shard that failed in build or test may still have
	# completed a good metadata fetch, and that fingerprint is worth keeping
	# for the next run.
	- name: Save checksum fallback store
	if: always()
	uses: actions/cache/save@v5
	with:
	path: ${{ runner.temp }}/checksum-fallback
	key: checksum-lkg-${{ matrix.context }}-${{ matrix.arch }}-${{ github.run_id }}-${{ github.run_attempt }}

	# A per-arch push leaves `<tag>-amd64` / `<tag>-arm64` in the registry; this
	# joins each pair into the real multi-arch `<tag>` (pass 1) and then points
	# every alias tag -- retagFrom entries such as `latest` -- at its source
	# (pass 2). Both are `imagetools create` calls: manifest-list writes against
	# images already in the registry, so nothing is pulled, rebuilt or
	# re-uploaded, which is why 15 minutes is generous.
	merge:
	needs: test
	if: github.event_name == 'push' && github.ref == 'refs/heads/master'
	runs-on: ubuntu-latest
	timeout-minutes: 15
	steps:
	- uses: actions/checkout@v7

	- name: Log in to Docker Hub
	uses: docker/login-action@v4
	with:
	username: ${{ secrets.DOCKERHUB_USERNAME }}
	password: ${{ secrets.DOCKERHUB_TOKEN }}

	# Done-stamps persisted across re-run attempts of this commit, same idea
	# as the integration-test stamps: `imagetools create` re-reads its
	# sources and re-writes the target even for an already-merged tag, so
	# without stamps a rate-limited merge can never converge -- every retry
	# redoes all ~110 tags and dies at the same quota depth. With them, each
	# retry only pays for the tags the previous attempt did not land.
	- name: Restore merge stamps
	uses: actions/cache/restore@v5
	with:
	path: ${{ runner.temp }}/merge-stamps
	key: merge-stamps-${{ github.sha }}-${{ github.run_attempt }}
	restore-keys: merge-stamps-${{ github.sha }}-

	- name: Merge per-arch manifests
	env:
	MERGE_STAMP_DIR: ${{ runner.temp }}/merge-stamps
	run: bin/merge-manifests

	- name: Save merge stamps
	if: always()
	uses: actions/cache/save@v5
	with:
	path: ${{ runner.temp }}/merge-stamps
	key: merge-stamps-${{ github.sha }}-${{ github.run_attempt }}

Navigation Menu

Upgrade pagila to current upstream with pgvector support #38

Workflow file

Upgrade pagila to current upstream with pgvector support #38

Uh oh!

Workflow file for this run