From b81025975a5040469565b5c45dde3f23f8ad9162 Mon Sep 17 00:00:00 2001 From: Artem Nikitin Date: Sat, 8 Aug 2026 12:46:40 +0200 Subject: [PATCH 1/5] Publish every architecture to one bucket per cloud Images now upload under an / key prefix instead of to a separate bucket per architecture. The Firework agent resolves that prefix from the node it runs on, so one bucket per cloud serves every architecture and a mixed-architecture fleet needs no extra configuration. The prefix uses the Go architecture vocabulary that TARGET_PLATFORM already carries, not the AWS x86_64 spelling the agent would never look under, and an unrecognised platform is rejected rather than published without a prefix. This removes the per-architecture bucket resolution added for the amd64 default: with the architecture in the key there is nothing to resolve, both matrix legs upload unconditionally, and the warning about an architecture that publishes nothing no longer describes a reachable state. Because both architectures previously produced identical object names, publishing them to one bucket only became safe once the key carried the architecture. The agent change must therefore ship after this one; docs/ci-pipeline.md documents the ordering and the one-time full re-download at cutover. Co-Authored-By: Claude Opus 5 --- .github/workflows/build-images.yaml | 81 ++++++---------------------- AGENTS.md | 8 +-- Makefile | 8 +-- docs/ci-pipeline.md | 82 ++++++++++++++++++----------- scripts/push-images.sh | 27 ++++++++-- scripts/test-push-images.sh | 41 +++++++++++---- 6 files changed, 129 insertions(+), 118 deletions(-) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index b660fc9..bc1af77 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -153,64 +153,22 @@ jobs: COMPARE_BASE_SHA: ${{ env.COMPARE_BASE_SHA }} run: make build - - name: Resolve upload buckets - if: env.SHOULD_UPLOAD == 'true' - env: - S3_IMAGES_BUCKET: ${{ vars.S3_IMAGES_BUCKET }} - S3_IMAGES_BUCKET_ARM64: ${{ vars.S3_IMAGES_BUCKET_ARM64 }} - S3_IMAGES_BUCKET_AMD64: ${{ vars.S3_IMAGES_BUCKET_AMD64 }} - GCS_IMAGES_BUCKET: ${{ vars.GCS_IMAGES_BUCKET }} - GCS_IMAGES_BUCKET_ARM64: ${{ vars.GCS_IMAGES_BUCKET_ARM64 }} - GCS_IMAGES_BUCKET_AMD64: ${{ vars.GCS_IMAGES_BUCKET_AMD64 }} - run: | - case "${{ matrix.target_arch }}" in - arm64) - S3_TARGET="${S3_IMAGES_BUCKET_ARM64:-}" - GCS_TARGET="${GCS_IMAGES_BUCKET_ARM64:-}" - ;; - amd64) - S3_TARGET="${S3_IMAGES_BUCKET_AMD64:-${S3_IMAGES_BUCKET:-}}" - GCS_TARGET="${GCS_IMAGES_BUCKET_AMD64:-${GCS_IMAGES_BUCKET:-}}" - ;; - esac - - # Publishing an architecture nobody consumes is intentional and cheap; - # silently publishing nothing is how an architecture mismatch reaches - # a running deployment. Make the no-op loud. - if [ -z "$S3_TARGET" ] && [ -z "$GCS_TARGET" ]; then - echo "::warning title=No ${{ matrix.target_arch }} image bucket configured::The ${{ matrix.target_arch }} build will produce rootfs images and publish none of them. Set the ${{ matrix.target_arch }} bucket variables to publish it." - fi - - echo "S3_IMAGES_BUCKET=$S3_TARGET" >> "$GITHUB_ENV" - echo "GCS_IMAGES_BUCKET=$GCS_TARGET" >> "$GITHUB_ENV" - + # Both architectures publish to the same bucket per cloud, separated by an + # / key prefix that push-images.sh derives from TARGET_PLATFORM. The + # agent reads the prefix matching its own node, so there is nothing + # per-architecture to resolve here. - name: Upload images to S3 - # Gated the same way as the GCS credentials below, so the arm64 leg is - # not handed AWS credentials for an upload it will never perform. - if: >- - env.SHOULD_UPLOAD == 'true' && - ( - (matrix.target_arch == 'arm64' && vars.S3_IMAGES_BUCKET_ARM64 != '') || - (matrix.target_arch == 'amd64' && (vars.S3_IMAGES_BUCKET_AMD64 != '' || vars.S3_IMAGES_BUCKET != '')) - ) + if: env.SHOULD_UPLOAD == 'true' && vars.S3_IMAGES_BUCKET != '' env: + TARGET_PLATFORM: ${{ matrix.target_platform }} + S3_IMAGES_BUCKET: ${{ vars.S3_IMAGES_BUCKET }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ vars.AWS_REGION }} - run: | - if [ -z "${S3_IMAGES_BUCKET:-}" ]; then - echo "Skipping S3 upload for ${{ matrix.target_platform }}; no bucket configured" - exit 0 - fi - make push-s3 + run: make push-s3 - name: Authenticate to GCP - if: >- - env.SHOULD_UPLOAD == 'true' && - ( - (matrix.target_arch == 'arm64' && vars.GCS_IMAGES_BUCKET_ARM64 != '') || - (matrix.target_arch == 'amd64' && (vars.GCS_IMAGES_BUCKET_AMD64 != '' || vars.GCS_IMAGES_BUCKET != '')) - ) + if: env.SHOULD_UPLOAD == 'true' && vars.GCS_IMAGES_BUCKET != '' uses: google-github-actions/auth@v3 with: workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} @@ -218,17 +176,10 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Upload images to GCS - # Matches the credentials step above; without this the arm64 leg runs an - # upload step that can only no-op, having never authenticated. - if: >- - env.SHOULD_UPLOAD == 'true' && - ( - (matrix.target_arch == 'arm64' && vars.GCS_IMAGES_BUCKET_ARM64 != '') || - (matrix.target_arch == 'amd64' && (vars.GCS_IMAGES_BUCKET_AMD64 != '' || vars.GCS_IMAGES_BUCKET != '')) - ) - run: | - if [ -z "${GCS_IMAGES_BUCKET:-}" ]; then - echo "Skipping GCS upload for ${{ matrix.target_platform }}; no bucket configured" - exit 0 - fi - make push-gcs + # Gated identically to the credentials step above, so this never runs + # without having authenticated. + if: env.SHOULD_UPLOAD == 'true' && vars.GCS_IMAGES_BUCKET != '' + env: + TARGET_PLATFORM: ${{ matrix.target_platform }} + GCS_IMAGES_BUCKET: ${{ vars.GCS_IMAGES_BUCKET }} + run: make push-gcs diff --git a/AGENTS.md b/AGENTS.md index 9f345d6..ba4d4d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Project -This is the example GitOps input repo for Firework. It defines tenant service YAML and config overlays used to build Firecracker-ready rootfs images and publish them to S3 and GCS via architecture-specific buckets. Both amd64 and ARM64 are built; amd64 is the published default because both data planes run x86_64 nodes, and ARM64 publishing is opt-in via the `*_ARM64` bucket variables. Public routing is provider-neutral via `metadata.subdomain`; there is no provider-specific runtime config tree. +This is the example GitOps input repo for Firework. It defines tenant service YAML and config overlays used to build Firecracker-ready rootfs images and publish them to S3 and GCS. Both amd64 and ARM64 are built and published, to one bucket per cloud, under an `/` key prefix; the agent resolves the prefix from the node it runs on. Public routing is provider-neutral via `metadata.subdomain`; there is no provider-specific runtime config tree. ## Layout @@ -13,8 +13,8 @@ This is the example GitOps input repo for Firework. It defines tenant service YA - `configs//` and `configs/-/`: rootfs overlays; tenant-specific overlays take precedence. - `scripts/build-images.sh`: resolves `fc-init` and builds tenant rootfs images. Skips a tenant service when its inputs (own YAML, shared/tenant overlays) are unchanged since `COMPARE_BASE_SHA`, unless `FORCE_REBUILD=true` or a shared pipeline file changed (see workflow for how these are set in CI). - `scripts/docker-to-rootfs.sh`: converts Docker images into ext4 rootfs images. -- `scripts/push-images.sh`: uploads generated rootfs images to the object store named by its `s3`/`gcs` backend argument. -- `scripts/test-push-images.sh`: regression checks for that backend selection. +- `scripts/push-images.sh`: uploads generated rootfs images to the object store named by its `s3`/`gcs` backend argument, under an `/` key prefix derived from `TARGET_PLATFORM`. +- `scripts/test-push-images.sh`: regression checks for that backend selection and the architecture prefix. - `scripts/fc-init/`: fallback bundled `fc-init` source for CI. ## Conventions @@ -30,7 +30,7 @@ For YAML-only changes, inspect schema consistency against the main repo docs. For image pipeline changes, validate: - `shellcheck scripts/docker-to-rootfs.sh` -- `bash scripts/test-push-images.sh` when touching `push-images.sh`, the `push-*` Makefile targets, or the workflow's bucket resolution. CI exports both bucket variables, so the backend must be selected explicitly rather than inferred. +- `bash scripts/test-push-images.sh` when touching `push-images.sh`, the `push-*` Makefile targets, or the workflow's upload steps. CI exports both bucket variables, so the backend must be selected explicitly rather than inferred, and `TARGET_PLATFORM` must reach the push targets or images publish under the wrong architecture prefix. - A targeted local rootfs build when Docker, `jq`, `mkfs.ext4`, and a linux/amd64 `fc-init` are available. For CI-equivalent validation, run `make build`, but skip `make push`. diff --git a/Makefile b/Makefile index 3f456c0..cd7846b 100644 --- a/Makefile +++ b/Makefile @@ -11,11 +11,13 @@ build-amd64: build-arm64: TARGET_PLATFORM=linux/arm64 bash ./scripts/build-images.sh +# TARGET_PLATFORM selects the / key prefix images are published under, so +# it must match the platform they were built for. push: - bash ./scripts/push-images.sh + TARGET_PLATFORM="$(TARGET_PLATFORM)" bash ./scripts/push-images.sh push-s3: - S3_IMAGES_BUCKET="$(S3_IMAGES_BUCKET)" bash ./scripts/push-images.sh s3 + TARGET_PLATFORM="$(TARGET_PLATFORM)" S3_IMAGES_BUCKET="$(S3_IMAGES_BUCKET)" bash ./scripts/push-images.sh s3 push-gcs: - GCS_IMAGES_BUCKET="$(GCS_IMAGES_BUCKET)" bash ./scripts/push-images.sh gcs + TARGET_PLATFORM="$(TARGET_PLATFORM)" GCS_IMAGES_BUCKET="$(GCS_IMAGES_BUCKET)" bash ./scripts/push-images.sh gcs diff --git a/docs/ci-pipeline.md b/docs/ci-pipeline.md index 7581fc6..8677f6f 100644 --- a/docs/ci-pipeline.md +++ b/docs/ci-pipeline.md @@ -8,9 +8,9 @@ It builds the tenant rootfs images twice, once for `linux/arm64` and once for On pushes to `main`, on the weekly schedule, and on manual dispatch (when run against `main`), each matrix build job publishes its architecture to the configured S3 bucket and, when configured, authenticates to GCP in the same job -and uploads that architecture to its GCS bucket. Keep the buckets -architecture-specific: the generated `*-rootfs.ext4` filenames are the same -across architectures, so sharing one bucket would cause overwrites. +and uploads that architecture to its GCS bucket. Both architectures share one +bucket per cloud: the generated `*-rootfs.ext4` filenames are identical across +architectures, so each is stored under an `/` key prefix. ## Change-aware builds @@ -37,39 +37,57 @@ before this feature existed. ## Bucket configuration -Both legacy variables mean the amd64 bucket, because both providers default to -x86_64 nodes: `S3_IMAGES_BUCKET` is the amd64 S3 bucket and `GCS_IMAGES_BUCKET` -is the amd64 GCS bucket. The explicit `*_AMD64` names are still supported and -take precedence when set. Publishing the arm64 build requires opting in with -`S3_IMAGES_BUCKET_ARM64` or `GCS_IMAGES_BUCKET_ARM64`; without them the arm64 -build still runs but uploads nothing. +One bucket per cloud holds every architecture: `S3_IMAGES_BUCKET` and +`GCS_IMAGES_BUCKET`. There are no per-architecture bucket variables. Both matrix +legs upload to the same bucket, under a key prefix taken from the platform they +were built for: -Resolved upload targets per build: +```text +/ + amd64/tenant-1-kibana-rootfs.ext4 + arm64/tenant-1-kibana-rootfs.ext4 +``` -| Build | S3 bucket | GCS bucket | -| --- | --- | --- | -| amd64 | `S3_IMAGES_BUCKET_AMD64`, else `S3_IMAGES_BUCKET` | `GCS_IMAGES_BUCKET_AMD64`, else `GCS_IMAGES_BUCKET` | -| arm64 | `S3_IMAGES_BUCKET_ARM64` | `GCS_IMAGES_BUCKET_ARM64` | +The prefix uses the Go architecture vocabulary (`amd64`, `arm64`) that +`TARGET_PLATFORM` already carries — deliberately not the AWS `x86_64` spelling, +which the agent would never look under. `push-images.sh` rejects an +unrecognised `TARGET_PLATFORM` rather than publishing without a prefix. -Both variables are exported for the amd64 build, so `push-images.sh` takes an +The Firework agent resolves the prefix from the architecture of the node it runs +on, so a node can only ever fetch images built for itself. Host and guest +architecture must match; before this layout a mismatch surfaced only at microVM +start, as a guest kernel panic. It now fails at image sync with a missing-object +error naming the key. + +This also means a mixed-architecture fleet needs no extra configuration: node +configs carry no architecture, so one desired state serves both. + +Both bucket variables are exported for every build, so `push-images.sh` takes an explicit `s3` or `gcs` backend argument rather than inferring one from whichever -bucket is set. `make push-s3` and `make push-gcs` pass it. Inference is still -accepted when exactly one bucket variable is set, and errors when both are, so -a publish can never silently go to the wrong object store. -`scripts/test-push-images.sh` covers this and runs in CI. - -Host and guest architecture must match, and a mismatch fails at microVM start -rather than at deploy time. The AWS data plane in -`firework-deployment-example` now defaults to x86_64 nodes using nested -virtualization rather than bare-metal Graviton, so it consumes the amd64 rootfs -images; the GCP data plane has always been x86_64. - -`S3_IMAGES_BUCKET` previously meant the arm64 S3 bucket, so an existing -deployment that keeps its value will now receive amd64 images in that same -bucket, replacing the arm64 objects under identical names. That is intended for -the default x86_64 AWS node. A deployment that still runs Graviton nodes -(`node_ami_architecture = "arm64"`) must set `S3_IMAGES_BUCKET_ARM64` and point -`s3_images_bucket_id` at that bucket instead. +bucket is set. `make push-s3` and `make push-gcs` pass it, along with +`TARGET_PLATFORM`. Inference is still accepted when exactly one bucket variable +is set, and errors when both are, so a publish can never silently go to the +wrong object store. `scripts/test-push-images.sh` covers this and runs in CI. + +### Migrating from per-architecture buckets + +Objects previously sat at the bucket root, and the agent read them there. The +agent change that reads `/` keys must not ship first, or every node fails +with a missing image. + +1. Merge this repository's change and run `workflow_dispatch` with + `force_rebuild = true`. Change-aware builds only publish services that + changed, so without a forced run the new prefixes stay incomplete. +2. Confirm both `amd64/` and `arm64/` prefixes are populated in each bucket. +3. Roll out agents that resolve arch-prefixed keys. +4. Delete the flat objects at the bucket root. + +Un-upgraded agents keep reading the frozen flat objects until they are replaced, +so the intermediate state is safe. Expect one full re-download per node at +cutover: the write-token sidecars survive, but a republished object under a new +key carries a new token. A deployment that publishes kernels to the bucket +rather than baking them into the node image must republish those under `/` +too. ## CI config validation diff --git a/scripts/push-images.sh b/scripts/push-images.sh index e7c246d..86b761c 100644 --- a/scripts/push-images.sh +++ b/scripts/push-images.sh @@ -9,9 +9,16 @@ # silent preference — CI exports both, so inference cannot # distinguish "push to S3" from "push to GCS" on its own. # +# One bucket per cloud holds every architecture. Objects are uploaded under an +# / prefix, and the Firework agent reads the prefix matching the node it +# runs on. The prefix uses the Go architecture vocabulary (amd64, arm64) that +# TARGET_PLATFORM already carries — not the AWS x86_64 spelling, which would be +# a silent 404 on the agent side. +# # Environment: # S3_IMAGES_BUCKET destination bucket for the s3 backend # GCS_IMAGES_BUCKET destination bucket for the gcs backend +# TARGET_PLATFORM platform the images were built for; sets the key prefix set -euo pipefail @@ -21,6 +28,18 @@ cd "$REPO_ROOT" BACKEND="${1:-}" +TARGET_PLATFORM="${TARGET_PLATFORM:-linux/amd64}" +case "$TARGET_PLATFORM" in + linux/amd64 | linux/arm64) + TARGET_ARCH="${TARGET_PLATFORM##*/}" + ;; + *) + echo "ERROR: unsupported target platform: $TARGET_PLATFORM" >&2 + echo "Supported platforms: linux/amd64, linux/arm64" >&2 + exit 1 + ;; +esac + push_s3() { if [ -z "${S3_IMAGES_BUCKET:-}" ]; then echo "ERROR: S3_IMAGES_BUCKET must be set for the s3 backend" >&2 @@ -28,8 +47,8 @@ push_s3() { fi for ext4 in *-rootfs.ext4; do [ -f "$ext4" ] || continue - echo "Uploading $ext4 to s3://${S3_IMAGES_BUCKET}/${ext4}" - aws s3 cp "$ext4" "s3://${S3_IMAGES_BUCKET}/${ext4}" + echo "Uploading $ext4 to s3://${S3_IMAGES_BUCKET}/${TARGET_ARCH}/${ext4}" + aws s3 cp "$ext4" "s3://${S3_IMAGES_BUCKET}/${TARGET_ARCH}/${ext4}" done } @@ -40,8 +59,8 @@ push_gcs() { fi for ext4 in *-rootfs.ext4; do [ -f "$ext4" ] || continue - echo "Uploading $ext4 to gs://${GCS_IMAGES_BUCKET}/${ext4}" - gcloud storage cp "$ext4" "gs://${GCS_IMAGES_BUCKET}/${ext4}" + echo "Uploading $ext4 to gs://${GCS_IMAGES_BUCKET}/${TARGET_ARCH}/${ext4}" + gcloud storage cp "$ext4" "gs://${GCS_IMAGES_BUCKET}/${TARGET_ARCH}/${ext4}" done } diff --git a/scripts/test-push-images.sh b/scripts/test-push-images.sh index 319b13b..bc4f370 100755 --- a/scripts/test-push-images.sh +++ b/scripts/test-push-images.sh @@ -1,12 +1,18 @@ #!/usr/bin/env bash -# Regression checks for scripts/push-images.sh backend selection. +# Regression checks for scripts/push-images.sh backend selection and the +# architecture key prefix. # -# CI exports both S3_IMAGES_BUCKET and GCS_IMAGES_BUCKET for the amd64 build, so +# CI exports both S3_IMAGES_BUCKET and GCS_IMAGES_BUCKET for every build, so # `make push-s3` must upload to S3 even though a GCS bucket is also configured. # An earlier version inferred the backend and preferred GCS, which silently sent # the AWS images to GCS and left the S3 bucket untouched. # +# One bucket per cloud holds every architecture, so the destination key must +# carry an / prefix derived from TARGET_PLATFORM. Publishing to the wrong +# prefix — or to none — is invisible until a node boots a guest built for +# another architecture. +# # Runs the real Makefile targets with `aws` and `gcloud` stubbed on PATH. set -euo pipefail @@ -72,20 +78,35 @@ run_case() { # The regression: both buckets set, as the amd64 CI leg exports them. run_case "push-s3 uploads to S3 when both buckets are set" \ - push-s3 "aws s3 cp demo-rootfs.ext4 s3://s3-bucket/demo-rootfs.ext4" \ - S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket + push-s3 "aws s3 cp demo-rootfs.ext4 s3://s3-bucket/amd64/demo-rootfs.ext4" \ + TARGET_PLATFORM=linux/amd64 S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket run_case "push-gcs uploads to GCS when both buckets are set" \ - push-gcs "gcloud storage cp demo-rootfs.ext4 gs://gcs-bucket/demo-rootfs.ext4" \ - S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket + push-gcs "gcloud storage cp demo-rootfs.ext4 gs://gcs-bucket/amd64/demo-rootfs.ext4" \ + TARGET_PLATFORM=linux/amd64 S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket run_case "push-s3 uploads to S3 when only the S3 bucket is set" \ - push-s3 "aws s3 cp demo-rootfs.ext4 s3://s3-bucket/demo-rootfs.ext4" \ - S3_IMAGES_BUCKET=s3-bucket + push-s3 "aws s3 cp demo-rootfs.ext4 s3://s3-bucket/amd64/demo-rootfs.ext4" \ + TARGET_PLATFORM=linux/amd64 S3_IMAGES_BUCKET=s3-bucket run_case "push-gcs uploads to GCS when only the GCS bucket is set" \ - push-gcs "gcloud storage cp demo-rootfs.ext4 gs://gcs-bucket/demo-rootfs.ext4" \ - GCS_IMAGES_BUCKET=gcs-bucket + push-gcs "gcloud storage cp demo-rootfs.ext4 gs://gcs-bucket/amd64/demo-rootfs.ext4" \ + GCS_IMAGES_BUCKET=gcs-bucket TARGET_PLATFORM=linux/amd64 + +# The arm64 leg must land under its own prefix; sharing one bucket makes a +# wrong prefix an overwrite of the other architecture. +run_case "push-s3 publishes the arm64 build under the arm64 prefix" \ + push-s3 "aws s3 cp demo-rootfs.ext4 s3://s3-bucket/arm64/demo-rootfs.ext4" \ + TARGET_PLATFORM=linux/arm64 S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket + +run_case "push-gcs publishes the arm64 build under the arm64 prefix" \ + push-gcs "gcloud storage cp demo-rootfs.ext4 gs://gcs-bucket/arm64/demo-rootfs.ext4" \ + TARGET_PLATFORM=linux/arm64 S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket + +# An unrecognised platform must fail rather than invent a prefix: the agent +# would look under amd64/ or arm64/ and find nothing. +run_case "push-s3 fails on an unsupported target platform" \ + push-s3 FAIL TARGET_PLATFORM=linux/riscv64 S3_IMAGES_BUCKET=s3-bucket run_case "push-s3 fails when the S3 bucket is unset" \ push-s3 FAIL GCS_IMAGES_BUCKET=gcs-bucket From fe526cf370eb32456af09c976bee2ccabb203ff0 Mon Sep 17 00:00:00 2001 From: Artem Nikitin Date: Sat, 8 Aug 2026 12:49:42 +0200 Subject: [PATCH 2/5] Assert the workflow passes TARGET_PLATFORM to the upload steps The backend cases run the Makefile directly and never see the workflow, so the env wiring that gives each matrix leg its architecture had no coverage. That gap matters more now than it would have before: both architectures publish to one bucket under identical object names, so an upload step missing TARGET_PLATFORM makes the arm64 leg publish under amd64/ and overwrite the amd64 images. Nothing fails until a node boots a guest built for the wrong architecture. Verified by removing the env line and watching the check fail. Co-Authored-By: Claude Opus 5 --- scripts/test-push-images.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/scripts/test-push-images.sh b/scripts/test-push-images.sh index bc4f370..1cd4f24 100755 --- a/scripts/test-push-images.sh +++ b/scripts/test-push-images.sh @@ -114,6 +114,28 @@ run_case "push-s3 fails when the S3 bucket is unset" \ run_case "push without a backend fails when both buckets are set" \ push FAIL S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket +# The cases above prove push-images.sh honours TARGET_PLATFORM, but they run the +# Makefile directly and never see the workflow. Both architectures now publish to +# one bucket under identical object names, so a missing TARGET_PLATFORM on an +# upload step makes the arm64 leg publish to amd64/ and overwrite the amd64 +# images — silently, until a node boots a guest built for the wrong architecture. +# Assert the wiring exists. +WORKFLOW="$REPO_ROOT/.github/workflows/build-images.yaml" +for step in push-s3 push-gcs; do + if awk -v want="make $step" ' + /^ - name:/ { in_step = 1; seen_platform = 0 } + /TARGET_PLATFORM:/ { if (in_step) seen_platform = 1 } + $0 ~ ("run: " want) { if (seen_platform) { found = 1 } } + END { exit !found } + ' "$WORKFLOW"; then + echo "ok: workflow passes TARGET_PLATFORM to the $step step" + else + echo "FAIL: the workflow step running 'make $step' does not set TARGET_PLATFORM" + echo " without it the arm64 build publishes under the amd64 prefix" + FAILURES=$((FAILURES + 1)) + fi +done + if [ "$FAILURES" -ne 0 ]; then echo "$FAILURES check(s) failed" >&2 exit 1 From 5a663c7ac8f95bef86769ed3e3dbdfaf199aa308 Mon Sep 17 00:00:00 2001 From: Artem Nikitin Date: Sat, 8 Aug 2026 17:02:08 +0200 Subject: [PATCH 3/5] Match the matrix value, require TARGET_PLATFORM, fix stale prose The workflow assertion only proved the TARGET_PLATFORM key was present in each upload step. Hardcoding the value passed the whole suite while pinning both matrix legs to one architecture, which is exactly the overwrite the check exists to catch. It now matches the matrix expression, verified against that bypass. push-images.sh no longer defaults TARGET_PLATFORM. build-arm64 sets it inside its own recipe while the push targets read it from the make environment, so `make build-arm64 && make push-s3` published arm64 images under amd64/ and overwrote them in the now-shared bucket. The Makefile default is removed too: build-images.sh already defaults on its own, so `make build` is unchanged while an unset push fails loudly. CI always sets it explicitly. The migration runbook now diffs the flat and prefixed object sets rather than asking for a visual check, because a node serving a cached image will not report a gap, and step 4 deletes the flat objects. README.md still described architecture-specific buckets with opt-in ARM64, contradicting AGENTS.md in the same change. Co-Authored-By: Claude Opus 5 --- Makefile | 4 +++- README.md | 2 +- docs/ci-pipeline.md | 33 +++++++++++++++++++++++++++------ scripts/push-images.sh | 12 +++++++++++- scripts/test-push-images.sh | 31 ++++++++++++++++++++++--------- 5 files changed, 64 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index cd7846b..f3f0a18 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,6 @@ -TARGET_PLATFORM ?= linux/amd64 +# Intentionally undefaulted. build-images.sh defaults to linux/amd64 on its own, +# while push-images.sh requires an explicit value: defaulting here would make +# `make build-arm64 && make push-s3` publish arm64 images under the amd64 prefix. .PHONY: build build-amd64 build-arm64 push push-s3 push-gcs diff --git a/README.md b/README.md index 2911429..bc5291d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ > This is an example deployment intended for demonstration and learning purposes only. It is not hardened, audited, etc. -Example GitOps repository for [Firework](https://github.com/artemnikitin/firework), focused on building Firecracker-ready rootfs images and publishing them to S3 and GCS via architecture-specific buckets. Both X86 and ARM64 images are built; X86 is published by default, because both the AWS and GCP data planes run x86_64 nodes, and ARM64 publishing is opt-in. +Example GitOps repository for [Firework](https://github.com/artemnikitin/firework), focused on building Firecracker-ready rootfs images and publishing them to S3 and GCS. Both x86 and ARM64 images are built and published, to one bucket per cloud, under an `/` key prefix. The Firework agent reads the prefix matching the node it runs on, so a single bucket serves a mixed-architecture fleet. ## Related Repositories diff --git a/docs/ci-pipeline.md b/docs/ci-pipeline.md index 8677f6f..55dafda 100644 --- a/docs/ci-pipeline.md +++ b/docs/ci-pipeline.md @@ -56,8 +56,13 @@ unrecognised `TARGET_PLATFORM` rather than publishing without a prefix. The Firework agent resolves the prefix from the architecture of the node it runs on, so a node can only ever fetch images built for itself. Host and guest architecture must match; before this layout a mismatch surfaced only at microVM -start, as a guest kernel panic. It now fails at image sync with a missing-object -error naming the key. +start, as a guest kernel panic. + +A missing image now fails at sync with an error naming the key — but only on a +node that has no local copy. The agent falls back to a cached image whenever an +object is absent, logging at debug level, so a node that already holds images +from the previous layout stays quiet. Verify bucket contents directly rather +than waiting for nodes to report a gap. This also means a mixed-architecture fleet needs no extra configuration: node configs carry no architecture, so one desired state serves both. @@ -72,15 +77,31 @@ wrong object store. `scripts/test-push-images.sh` covers this and runs in CI. ### Migrating from per-architecture buckets Objects previously sat at the bucket root, and the agent read them there. The -agent change that reads `/` keys must not ship first, or every node fails -with a missing image. +agent change that reads `/` keys must not ship first: a freshly built node +would find nothing under its prefix and fail. An existing node keeps running on +its cached images, so the damage is uneven and easy to miss — which is why the +verification below is a diff rather than a glance. 1. Merge this repository's change and run `workflow_dispatch` with `force_rebuild = true`. Change-aware builds only publish services that changed, so without a forced run the new prefixes stay incomplete. -2. Confirm both `amd64/` and `arm64/` prefixes are populated in each bucket. +2. Diff the object sets, do not eyeball them. Every flat object must have a + counterpart under each architecture prefix that has nodes: + + ```bash + aws s3 ls "s3://$BUCKET/" | awk '{print $4}' | grep -v '/$' | sort > /tmp/flat + aws s3 ls "s3://$BUCKET/amd64/" | awk '{print $4}' | sort > /tmp/amd64 + comm -23 /tmp/flat /tmp/amd64 # must be empty before step 4 + ``` + + A forced rebuild that skipped a service — a build failure, or a service no + longer in the tenant set — leaves a gap here. Agents will not report it: + they serve the cached local copy. 3. Roll out agents that resolve arch-prefixed keys. -4. Delete the flat objects at the bucket root. +4. Delete the flat objects at the bucket root, only once step 2 shows no + difference. Deleting while a gap remains leaves nodes running an image that + exists nowhere in the bucket, which surfaces at the next node replacement, + long after the migration. Un-upgraded agents keep reading the frozen flat objects until they are replaced, so the intermediate state is safe. Expect one full re-download per node at diff --git a/scripts/push-images.sh b/scripts/push-images.sh index 86b761c..aca1709 100644 --- a/scripts/push-images.sh +++ b/scripts/push-images.sh @@ -28,7 +28,17 @@ cd "$REPO_ROOT" BACKEND="${1:-}" -TARGET_PLATFORM="${TARGET_PLATFORM:-linux/amd64}" +# Deliberately no default. `make build-arm64` sets TARGET_PLATFORM inside its +# own recipe, so a defaulted push would let `make build-arm64 && make push-s3` +# publish arm64 images under amd64/ and overwrite them in the now-shared bucket. +# CI always sets it explicitly, so requiring it costs nothing. +if [ -z "${TARGET_PLATFORM:-}" ]; then + echo "ERROR: TARGET_PLATFORM must be set; it selects the / key prefix" >&2 + echo "Pass the platform the images were built for, e.g.:" >&2 + echo " TARGET_PLATFORM=linux/arm64 make push-s3" >&2 + exit 1 +fi + case "$TARGET_PLATFORM" in linux/amd64 | linux/arm64) TARGET_ARCH="${TARGET_PLATFORM##*/}" diff --git a/scripts/test-push-images.sh b/scripts/test-push-images.sh index 1cd4f24..d9cdeea 100755 --- a/scripts/test-push-images.sh +++ b/scripts/test-push-images.sh @@ -109,10 +109,18 @@ run_case "push-s3 fails on an unsupported target platform" \ push-s3 FAIL TARGET_PLATFORM=linux/riscv64 S3_IMAGES_BUCKET=s3-bucket run_case "push-s3 fails when the S3 bucket is unset" \ - push-s3 FAIL GCS_IMAGES_BUCKET=gcs-bucket + push-s3 FAIL TARGET_PLATFORM=linux/amd64 GCS_IMAGES_BUCKET=gcs-bucket run_case "push without a backend fails when both buckets are set" \ - push FAIL S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket + push FAIL TARGET_PLATFORM=linux/amd64 S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket + +# The build-arm64 footgun: build-arm64 sets TARGET_PLATFORM in its own recipe, +# so a defaulted push would send arm64 images to the amd64 prefix. +run_case "push-s3 fails when TARGET_PLATFORM is unset" \ + push-s3 FAIL S3_IMAGES_BUCKET=s3-bucket + +run_case "push-gcs fails when TARGET_PLATFORM is unset" \ + push-gcs FAIL GCS_IMAGES_BUCKET=gcs-bucket # The cases above prove push-images.sh honours TARGET_PLATFORM, but they run the # Makefile directly and never see the workflow. Both architectures now publish to @@ -122,16 +130,21 @@ run_case "push without a backend fails when both buckets are set" \ # Assert the wiring exists. WORKFLOW="$REPO_ROOT/.github/workflows/build-images.yaml" for step in push-s3 push-gcs; do - if awk -v want="make $step" ' - /^ - name:/ { in_step = 1; seen_platform = 0 } - /TARGET_PLATFORM:/ { if (in_step) seen_platform = 1 } - $0 ~ ("run: " want) { if (seen_platform) { found = 1 } } + # Match the value, not just the key. A hardcoded TARGET_PLATFORM would keep + # the key present while pinning both matrix legs to one architecture, which + # is precisely the overwrite this guards against. + if awk -v want="run: make $step" ' + /^ - name:/ { in_step = 1; wired = 0 } + in_step && index($0, "TARGET_PLATFORM: ${{ matrix.target_platform }}") { wired = 1 } + in_step && index($0, want) { if (wired) found = 1 } END { exit !found } ' "$WORKFLOW"; then - echo "ok: workflow passes TARGET_PLATFORM to the $step step" + echo "ok: workflow passes the matrix architecture to the $step step" else - echo "FAIL: the workflow step running 'make $step' does not set TARGET_PLATFORM" - echo " without it the arm64 build publishes under the amd64 prefix" + echo "FAIL: the workflow step running 'make $step' does not set" + echo " TARGET_PLATFORM: \${{ matrix.target_platform }}" + echo " Without the matrix value both legs publish under one prefix," + echo " so the arm64 build overwrites the amd64 images." FAILURES=$((FAILURES + 1)) fi done From f6d39b1fcd6b9f3bc7ba2255c49ad23da50b2b97 Mon Sep 17 00:00:00 2001 From: Artem Nikitin Date: Sun, 9 Aug 2026 12:28:04 +0200 Subject: [PATCH 4/5] Restore the no-destination guard and drop the migration runbook Collapsing the per-architecture bucket resolution removed the warning that fired when a publishing run had no bucket configured. Without it, main, scheduled, and forced builds stay green while uploading nothing. A publishing run now fails outright when neither bucket variable is set, and warns per provider when only one is. The migration runbook is replaced with a short note about leftover root-level objects. There are no deployments on the previous layout, and the runbook had four defects that only mattered if someone followed it: it never told operators to repoint nodes that used a separate ARM64 bucket, it verified only amd64 and only S3, it required every flat object to have a prefixed counterpart while acknowledging some are intentionally obsolete, and its listing parsed `PRE /` rows into empty entries that made the pre-deletion check unreliable. The retained deletion snippet uses list-objects-v2 with a delimiter, so it returns root-level keys only and cannot produce those empty entries. Also corrects the claim that both bucket variables are exported for every build: each upload step now exports only its own provider's bucket, so the stated rationale for requiring an explicit backend argument no longer described the workflow. Co-Authored-By: Claude Opus 5 --- .github/workflows/build-images.yaml | 19 +++++++ docs/ci-pipeline.md | 81 ++++++++++++----------------- scripts/test-push-images.sh | 9 ++-- 3 files changed, 58 insertions(+), 51 deletions(-) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index bc1af77..6fae138 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -153,6 +153,25 @@ jobs: COMPARE_BASE_SHA: ${{ env.COMPARE_BASE_SHA }} run: make build + # A run that is supposed to publish but has no bucket configured would + # otherwise stay green while uploading nothing, which is how images + # silently stop reaching a deployment. Fail instead: on a publishing run, + # at least one destination must exist. + - name: Check a publish destination is configured + if: env.SHOULD_UPLOAD == 'true' + env: + S3_IMAGES_BUCKET: ${{ vars.S3_IMAGES_BUCKET }} + GCS_IMAGES_BUCKET: ${{ vars.GCS_IMAGES_BUCKET }} + run: | + if [ -z "${S3_IMAGES_BUCKET:-}" ] && [ -z "${GCS_IMAGES_BUCKET:-}" ]; then + echo "::error title=No image bucket configured::This run publishes ${{ matrix.target_arch }} images, but neither S3_IMAGES_BUCKET nor GCS_IMAGES_BUCKET is set. Set at least one repository variable." + exit 1 + fi + [ -n "${S3_IMAGES_BUCKET:-}" ] || \ + echo "::warning title=No S3 image bucket::${{ matrix.target_arch }} images will not be published to S3." + [ -n "${GCS_IMAGES_BUCKET:-}" ] || \ + echo "::warning title=No GCS image bucket::${{ matrix.target_arch }} images will not be published to GCS." + # Both architectures publish to the same bucket per cloud, separated by an # / key prefix that push-images.sh derives from TARGET_PLATFORM. The # agent reads the prefix matching its own node, so there is nothing diff --git a/docs/ci-pipeline.md b/docs/ci-pipeline.md index 55dafda..9f54197 100644 --- a/docs/ci-pipeline.md +++ b/docs/ci-pipeline.md @@ -58,57 +58,44 @@ on, so a node can only ever fetch images built for itself. Host and guest architecture must match; before this layout a mismatch surfaced only at microVM start, as a guest kernel panic. -A missing image now fails at sync with an error naming the key — but only on a -node that has no local copy. The agent falls back to a cached image whenever an -object is absent, logging at debug level, so a node that already holds images -from the previous layout stays quiet. Verify bucket contents directly rather -than waiting for nodes to report a gap. +A missing image fails at sync with an error naming the key — but only on a node +that has no local copy. The agent falls back to a cached image whenever an +object is absent, logging at debug level, so a node that already holds the image +stays quiet. Verify bucket contents directly rather than waiting for nodes to +report a gap. This also means a mixed-architecture fleet needs no extra configuration: node configs carry no architecture, so one desired state serves both. -Both bucket variables are exported for every build, so `push-images.sh` takes an -explicit `s3` or `gcs` backend argument rather than inferring one from whichever -bucket is set. `make push-s3` and `make push-gcs` pass it, along with -`TARGET_PLATFORM`. Inference is still accepted when exactly one bucket variable -is set, and errors when both are, so a publish can never silently go to the -wrong object store. `scripts/test-push-images.sh` covers this and runs in CI. - -### Migrating from per-architecture buckets - -Objects previously sat at the bucket root, and the agent read them there. The -agent change that reads `/` keys must not ship first: a freshly built node -would find nothing under its prefix and fail. An existing node keeps running on -its cached images, so the damage is uneven and easy to miss — which is why the -verification below is a diff rather than a glance. - -1. Merge this repository's change and run `workflow_dispatch` with - `force_rebuild = true`. Change-aware builds only publish services that - changed, so without a forced run the new prefixes stay incomplete. -2. Diff the object sets, do not eyeball them. Every flat object must have a - counterpart under each architecture prefix that has nodes: - - ```bash - aws s3 ls "s3://$BUCKET/" | awk '{print $4}' | grep -v '/$' | sort > /tmp/flat - aws s3 ls "s3://$BUCKET/amd64/" | awk '{print $4}' | sort > /tmp/amd64 - comm -23 /tmp/flat /tmp/amd64 # must be empty before step 4 - ``` - - A forced rebuild that skipped a service — a build failure, or a service no - longer in the tenant set — leaves a gap here. Agents will not report it: - they serve the cached local copy. -3. Roll out agents that resolve arch-prefixed keys. -4. Delete the flat objects at the bucket root, only once step 2 shows no - difference. Deleting while a gap remains leaves nodes running an image that - exists nowhere in the bucket, which surfaces at the next node replacement, - long after the migration. - -Un-upgraded agents keep reading the frozen flat objects until they are replaced, -so the intermediate state is safe. Expect one full re-download per node at -cutover: the write-token sidecars survive, but a republished object under a new -key carries a new token. A deployment that publishes kernels to the bucket -rather than baking them into the node image must republish those under `/` -too. +`push-images.sh` takes an explicit `s3` or `gcs` backend argument rather than +inferring one from whichever bucket is set, and `make push-s3` / `make push-gcs` +pass it along with `TARGET_PLATFORM`. Each workflow upload step now exports only +its own provider's bucket, so inference would happen to work — but a local run +or a future step that exports both must not be able to publish to the wrong +object store. Inference is therefore still accepted only when exactly one bucket +variable is set, and errors when both are. +`scripts/test-push-images.sh` covers this and runs in CI. + +### Objects from the previous layout + +Before this layout, objects sat at the bucket root and both the agent and node +bootstrap read them there. Nothing reads the bucket root now, so any root-level +`*-rootfs.ext4` left over from that layout is inert and can be deleted whenever +convenient: + +```bash +aws s3api list-objects-v2 --bucket "$BUCKET" --delimiter / \ + --query 'Contents[].Key' --output text | tr '\t' '\n' +``` + +`list-objects-v2` with `--delimiter /` returns only root-level keys, without the +`PRE /` rows that a plain `aws s3 ls` emits — those rows have no object +name in the column a naive parse would read, which silently produces empty +entries. + +Fresh nodes have no local image cache, so they depend on the prefixes being +populated before they boot. Run a `force_rebuild` dispatch after any change to +this pipeline and confirm both prefixes are present before deploying. ## CI config validation diff --git a/scripts/test-push-images.sh b/scripts/test-push-images.sh index d9cdeea..7b70ae3 100755 --- a/scripts/test-push-images.sh +++ b/scripts/test-push-images.sh @@ -3,10 +3,11 @@ # Regression checks for scripts/push-images.sh backend selection and the # architecture key prefix. # -# CI exports both S3_IMAGES_BUCKET and GCS_IMAGES_BUCKET for every build, so -# `make push-s3` must upload to S3 even though a GCS bucket is also configured. -# An earlier version inferred the backend and preferred GCS, which silently sent -# the AWS images to GCS and left the S3 bucket untouched. +# `make push-s3` must upload to S3 even when a GCS bucket is also configured in +# the environment. An earlier version inferred the backend and preferred GCS, +# which silently sent the AWS images to GCS and left the S3 bucket untouched. +# The workflow now exports only the relevant provider's bucket per upload step, +# but a local run can still have both set, so the guarantee is still tested. # # One bucket per cloud holds every architecture, so the destination key must # carry an / prefix derived from TARGET_PLATFORM. Publishing to the wrong From 1a14573544177cec454e81fae15ba83a45028f9d Mon Sep 17 00:00:00 2001 From: Artem Nikitin Date: Sun, 9 Aug 2026 12:41:47 +0200 Subject: [PATCH 5/5] Finish correcting the both-buckets-exported claim Three places still said CI exports both bucket variables. Each upload step now exports only its own provider's bucket, so the reason to require an explicit backend argument is a local run with both configured, not the workflow. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 2 +- scripts/push-images.sh | 5 +++-- scripts/test-push-images.sh | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ba4d4d2..d233dc8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,7 @@ For YAML-only changes, inspect schema consistency against the main repo docs. For image pipeline changes, validate: - `shellcheck scripts/docker-to-rootfs.sh` -- `bash scripts/test-push-images.sh` when touching `push-images.sh`, the `push-*` Makefile targets, or the workflow's upload steps. CI exports both bucket variables, so the backend must be selected explicitly rather than inferred, and `TARGET_PLATFORM` must reach the push targets or images publish under the wrong architecture prefix. +- `bash scripts/test-push-images.sh` when touching `push-images.sh`, the `push-*` Makefile targets, or the workflow's upload steps. Each workflow upload step exports only its own provider's bucket, but a local run can have both set, so the backend must be selected explicitly rather than inferred. `TARGET_PLATFORM` must also reach the push targets, or images publish under the wrong architecture prefix. - A targeted local rootfs build when Docker, `jq`, `mkfs.ext4`, and a linux/amd64 `fc-init` are available. For CI-equivalent validation, run `make build`, but skip `make push`. diff --git a/scripts/push-images.sh b/scripts/push-images.sh index aca1709..51225cd 100644 --- a/scripts/push-images.sh +++ b/scripts/push-images.sh @@ -6,8 +6,9 @@ # matching bucket variable is required and the other is ignored. # When omitted the backend is inferred from whichever single bucket # variable is set, and having both set is an error rather than a -# silent preference — CI exports both, so inference cannot -# distinguish "push to S3" from "push to GCS" on its own. +# silent preference: inference cannot distinguish "push to S3" from +# "push to GCS" on its own, and a caller with both configured would +# otherwise publish to whichever one the script happened to prefer. # # One bucket per cloud holds every architecture. Objects are uploaded under an # / prefix, and the Firework agent reads the prefix matching the node it diff --git a/scripts/test-push-images.sh b/scripts/test-push-images.sh index 7b70ae3..0a55937 100755 --- a/scripts/test-push-images.sh +++ b/scripts/test-push-images.sh @@ -77,7 +77,7 @@ run_case() { fi } -# The regression: both buckets set, as the amd64 CI leg exports them. +# The regression: both buckets set, as a local run with both configured has. run_case "push-s3 uploads to S3 when both buckets are set" \ push-s3 "aws s3 cp demo-rootfs.ext4 s3://s3-bucket/amd64/demo-rootfs.ext4" \ TARGET_PLATFORM=linux/amd64 S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket