Publish every architecture to one bucket per cloud

.github/workflows/build-images.yaml

@@ -153,82 +153,52 @@ jobs:
           COMPARE_BASE_SHA: ${{ env.COMPARE_BASE_SHA }}
         run: make build
 
-      - name: Resolve upload buckets
+      # A run that is supposed to publish but has no bucket configured would
+      # otherwise stay green while uploading nothing, which is how images
+      # silently stop reaching a deployment. Fail instead: on a publishing run,
+      # at least one destination must exist.
+      - name: Check a publish destination is configured
         if: env.SHOULD_UPLOAD == 'true'
         env:
           S3_IMAGES_BUCKET: ${{ vars.S3_IMAGES_BUCKET }}
-          S3_IMAGES_BUCKET_ARM64: ${{ vars.S3_IMAGES_BUCKET_ARM64 }}
-          S3_IMAGES_BUCKET_AMD64: ${{ vars.S3_IMAGES_BUCKET_AMD64 }}
           GCS_IMAGES_BUCKET: ${{ vars.GCS_IMAGES_BUCKET }}
-          GCS_IMAGES_BUCKET_ARM64: ${{ vars.GCS_IMAGES_BUCKET_ARM64 }}
-          GCS_IMAGES_BUCKET_AMD64: ${{ vars.GCS_IMAGES_BUCKET_AMD64 }}
         run: |
-          case "${{ matrix.target_arch }}" in
-            arm64)
-              S3_TARGET="${S3_IMAGES_BUCKET_ARM64:-}"
-              GCS_TARGET="${GCS_IMAGES_BUCKET_ARM64:-}"
-              ;;
-            amd64)
-              S3_TARGET="${S3_IMAGES_BUCKET_AMD64:-${S3_IMAGES_BUCKET:-}}"
-              GCS_TARGET="${GCS_IMAGES_BUCKET_AMD64:-${GCS_IMAGES_BUCKET:-}}"
-              ;;
-          esac
-
-          # Publishing an architecture nobody consumes is intentional and cheap;
-          # silently publishing nothing is how an architecture mismatch reaches
-          # a running deployment. Make the no-op loud.
-          if [ -z "$S3_TARGET" ] && [ -z "$GCS_TARGET" ]; then
-            echo "::warning title=No ${{ matrix.target_arch }} image bucket configured::The ${{ matrix.target_arch }} build will produce rootfs images and publish none of them. Set the ${{ matrix.target_arch }} bucket variables to publish it."
+          if [ -z "${S3_IMAGES_BUCKET:-}" ] && [ -z "${GCS_IMAGES_BUCKET:-}" ]; then
+            echo "::error title=No image bucket configured::This run publishes ${{ matrix.target_arch }} images, but neither S3_IMAGES_BUCKET nor GCS_IMAGES_BUCKET is set. Set at least one repository variable."
+            exit 1
           fi
-
-          echo "S3_IMAGES_BUCKET=$S3_TARGET" >> "$GITHUB_ENV"
-          echo "GCS_IMAGES_BUCKET=$GCS_TARGET" >> "$GITHUB_ENV"
-
+          [ -n "${S3_IMAGES_BUCKET:-}" ] || \
+            echo "::warning title=No S3 image bucket::${{ matrix.target_arch }} images will not be published to S3."
+          [ -n "${GCS_IMAGES_BUCKET:-}" ] || \
+            echo "::warning title=No GCS image bucket::${{ matrix.target_arch }} images will not be published to GCS."
+
+      # Both architectures publish to the same bucket per cloud, separated by an
+      # <arch>/ key prefix that push-images.sh derives from TARGET_PLATFORM. The
+      # agent reads the prefix matching its own node, so there is nothing
+      # per-architecture to resolve here.
       - name: Upload images to S3
-        # Gated the same way as the GCS credentials below, so the arm64 leg is
-        # not handed AWS credentials for an upload it will never perform.
-        if: >-
-          env.SHOULD_UPLOAD == 'true' &&
-          (
-            (matrix.target_arch == 'arm64' && vars.S3_IMAGES_BUCKET_ARM64 != '') ||
-            (matrix.target_arch == 'amd64' && (vars.S3_IMAGES_BUCKET_AMD64 != '' || vars.S3_IMAGES_BUCKET != ''))
-          )
+        if: env.SHOULD_UPLOAD == 'true' && vars.S3_IMAGES_BUCKET != ''
         env:
+          TARGET_PLATFORM: ${{ matrix.target_platform }}
+          S3_IMAGES_BUCKET: ${{ vars.S3_IMAGES_BUCKET }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: ${{ vars.AWS_REGION }}
-        run: |
-          if [ -z "${S3_IMAGES_BUCKET:-}" ]; then
-            echo "Skipping S3 upload for ${{ matrix.target_platform }}; no bucket configured"
-            exit 0
-          fi
-          make push-s3
+        run: make push-s3
 
       - name: Authenticate to GCP
-        if: >-
-          env.SHOULD_UPLOAD == 'true' &&
-          (
-            (matrix.target_arch == 'arm64' && vars.GCS_IMAGES_BUCKET_ARM64 != '') ||
-            (matrix.target_arch == 'amd64' && (vars.GCS_IMAGES_BUCKET_AMD64 != '' || vars.GCS_IMAGES_BUCKET != ''))
-          )
+        if: env.SHOULD_UPLOAD == 'true' && vars.GCS_IMAGES_BUCKET != ''
         uses: google-github-actions/auth@v3
         with:
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
           project_id: ${{ secrets.GCP_PROJECT_ID }}
 
       - name: Upload images to GCS
-        # Matches the credentials step above; without this the arm64 leg runs an
-        # upload step that can only no-op, having never authenticated.
-        if: >-
-          env.SHOULD_UPLOAD == 'true' &&
-          (
-            (matrix.target_arch == 'arm64' && vars.GCS_IMAGES_BUCKET_ARM64 != '') ||
-            (matrix.target_arch == 'amd64' && (vars.GCS_IMAGES_BUCKET_AMD64 != '' || vars.GCS_IMAGES_BUCKET != ''))
-          )
-        run: |
-          if [ -z "${GCS_IMAGES_BUCKET:-}" ]; then
-            echo "Skipping GCS upload for ${{ matrix.target_platform }}; no bucket configured"
-            exit 0
-          fi
-          make push-gcs
+        # Gated identically to the credentials step above, so this never runs
+        # without having authenticated.
+        if: env.SHOULD_UPLOAD == 'true' && vars.GCS_IMAGES_BUCKET != ''
+        env:
+          TARGET_PLATFORM: ${{ matrix.target_platform }}
+          GCS_IMAGES_BUCKET: ${{ vars.GCS_IMAGES_BUCKET }}
+        run: make push-gcs

AGENTS.md

@@ -2,7 +2,7 @@
 
 ## Project
 
-This is the example GitOps input repo for Firework. It defines tenant service YAML and config overlays used to build Firecracker-ready rootfs images and publish them to S3 and GCS via architecture-specific buckets. Both amd64 and ARM64 are built; amd64 is the published default because both data planes run x86_64 nodes, and ARM64 publishing is opt-in via the `*_ARM64` bucket variables. Public routing is provider-neutral via `metadata.subdomain`; there is no provider-specific runtime config tree.
+This is the example GitOps input repo for Firework. It defines tenant service YAML and config overlays used to build Firecracker-ready rootfs images and publish them to S3 and GCS. Both amd64 and ARM64 are built and published, to one bucket per cloud, under an `<arch>/` key prefix; the agent resolves the prefix from the node it runs on. Public routing is provider-neutral via `metadata.subdomain`; there is no provider-specific runtime config tree.
 
 ## Layout
 
@@ -13,8 +13,8 @@ This is the example GitOps input repo for Firework. It defines tenant service YA
 - `configs/<service>/` and `configs/<tenant>-<service>/`: rootfs overlays; tenant-specific overlays take precedence.
 - `scripts/build-images.sh`: resolves `fc-init` and builds tenant rootfs images. Skips a tenant service when its inputs (own YAML, shared/tenant overlays) are unchanged since `COMPARE_BASE_SHA`, unless `FORCE_REBUILD=true` or a shared pipeline file changed (see workflow for how these are set in CI).
 - `scripts/docker-to-rootfs.sh`: converts Docker images into ext4 rootfs images.
-- `scripts/push-images.sh`: uploads generated rootfs images to the object store named by its `s3`/`gcs` backend argument.
-- `scripts/test-push-images.sh`: regression checks for that backend selection.
+- `scripts/push-images.sh`: uploads generated rootfs images to the object store named by its `s3`/`gcs` backend argument, under an `<arch>/` key prefix derived from `TARGET_PLATFORM`.
+- `scripts/test-push-images.sh`: regression checks for that backend selection and the architecture prefix.
 - `scripts/fc-init/`: fallback bundled `fc-init` source for CI.
 
 ## Conventions
@@ -30,7 +30,7 @@ For YAML-only changes, inspect schema consistency against the main repo docs.
 For image pipeline changes, validate:
 
 - `shellcheck scripts/docker-to-rootfs.sh`
-- `bash scripts/test-push-images.sh` when touching `push-images.sh`, the `push-*` Makefile targets, or the workflow's bucket resolution. CI exports both bucket variables, so the backend must be selected explicitly rather than inferred.
+- `bash scripts/test-push-images.sh` when touching `push-images.sh`, the `push-*` Makefile targets, or the workflow's upload steps. Each workflow upload step exports only its own provider's bucket, but a local run can have both set, so the backend must be selected explicitly rather than inferred. `TARGET_PLATFORM` must also reach the push targets, or images publish under the wrong architecture prefix.
 - A targeted local rootfs build when Docker, `jq`, `mkfs.ext4`, and a linux/amd64 `fc-init` are available.
 
 For CI-equivalent validation, run `make build`, but skip `make push`.

Makefile

@@ -1,4 +1,6 @@
-TARGET_PLATFORM ?= linux/amd64
+# Intentionally undefaulted. build-images.sh defaults to linux/amd64 on its own,
+# while push-images.sh requires an explicit value: defaulting here would make
+# `make build-arm64 && make push-s3` publish arm64 images under the amd64 prefix.
 
 .PHONY: build build-amd64 build-arm64 push push-s3 push-gcs
 
@@ -11,11 +13,13 @@ build-amd64:
 build-arm64:
 	TARGET_PLATFORM=linux/arm64 bash ./scripts/build-images.sh
 
+# TARGET_PLATFORM selects the <arch>/ key prefix images are published under, so
+# it must match the platform they were built for.
 push:
-	bash ./scripts/push-images.sh
+	TARGET_PLATFORM="$(TARGET_PLATFORM)" bash ./scripts/push-images.sh
 
 push-s3:
-	S3_IMAGES_BUCKET="$(S3_IMAGES_BUCKET)" bash ./scripts/push-images.sh s3
+	TARGET_PLATFORM="$(TARGET_PLATFORM)" S3_IMAGES_BUCKET="$(S3_IMAGES_BUCKET)" bash ./scripts/push-images.sh s3
 
 push-gcs:
-	GCS_IMAGES_BUCKET="$(GCS_IMAGES_BUCKET)" bash ./scripts/push-images.sh gcs
+	TARGET_PLATFORM="$(TARGET_PLATFORM)" GCS_IMAGES_BUCKET="$(GCS_IMAGES_BUCKET)" bash ./scripts/push-images.sh gcs

README.md

@@ -2,7 +2,7 @@
 
 > This is an example deployment intended for demonstration and learning purposes only. It is not hardened, audited, etc.
 
-Example GitOps repository for [Firework](https://github.com/artemnikitin/firework), focused on building Firecracker-ready rootfs images and publishing them to S3 and GCS via architecture-specific buckets. Both X86 and ARM64 images are built; X86 is published by default, because both the AWS and GCP data planes run x86_64 nodes, and ARM64 publishing is opt-in.
+Example GitOps repository for [Firework](https://github.com/artemnikitin/firework), focused on building Firecracker-ready rootfs images and publishing them to S3 and GCS. Both x86 and ARM64 images are built and published, to one bucket per cloud, under an `<arch>/` key prefix. The Firework agent reads the prefix matching the node it runs on, so a single bucket serves a mixed-architecture fleet.
 
 ## Related Repositories
 

docs/ci-pipeline.md

@@ -8,9 +8,9 @@ It builds the tenant rootfs images twice, once for `linux/arm64` and once for
 On pushes to `main`, on the weekly schedule, and on manual dispatch (when run
 against `main`), each matrix build job publishes its architecture to the
 configured S3 bucket and, when configured, authenticates to GCP in the same job
-and uploads that architecture to its GCS bucket. Keep the buckets
-architecture-specific: the generated `*-rootfs.ext4` filenames are the same
-across architectures, so sharing one bucket would cause overwrites.
+and uploads that architecture to its GCS bucket. Both architectures share one
+bucket per cloud: the generated `*-rootfs.ext4` filenames are identical across
+architectures, so each is stored under an `<arch>/` key prefix.
 
 ## Change-aware builds
 
@@ -37,39 +37,65 @@ before this feature existed.
 
 ## Bucket configuration
 
-Both legacy variables mean the amd64 bucket, because both providers default to
-x86_64 nodes: `S3_IMAGES_BUCKET` is the amd64 S3 bucket and `GCS_IMAGES_BUCKET`
-is the amd64 GCS bucket. The explicit `*_AMD64` names are still supported and
-take precedence when set. Publishing the arm64 build requires opting in with
-`S3_IMAGES_BUCKET_ARM64` or `GCS_IMAGES_BUCKET_ARM64`; without them the arm64
-build still runs but uploads nothing.
-
-Resolved upload targets per build:
-
-| Build | S3 bucket | GCS bucket |
-| --- | --- | --- |
-| amd64 | `S3_IMAGES_BUCKET_AMD64`, else `S3_IMAGES_BUCKET` | `GCS_IMAGES_BUCKET_AMD64`, else `GCS_IMAGES_BUCKET` |
-| arm64 | `S3_IMAGES_BUCKET_ARM64` | `GCS_IMAGES_BUCKET_ARM64` |
-
-Both variables are exported for the amd64 build, so `push-images.sh` takes an
-explicit `s3` or `gcs` backend argument rather than inferring one from whichever
-bucket is set. `make push-s3` and `make push-gcs` pass it. Inference is still
-accepted when exactly one bucket variable is set, and errors when both are, so
-a publish can never silently go to the wrong object store.
+One bucket per cloud holds every architecture: `S3_IMAGES_BUCKET` and
+`GCS_IMAGES_BUCKET`. There are no per-architecture bucket variables. Both matrix
+legs upload to the same bucket, under a key prefix taken from the platform they
+were built for:
+
+```text
+<images-bucket>/
+  amd64/tenant-1-kibana-rootfs.ext4
+  arm64/tenant-1-kibana-rootfs.ext4
+```
+
+The prefix uses the Go architecture vocabulary (`amd64`, `arm64`) that
+`TARGET_PLATFORM` already carries — deliberately not the AWS `x86_64` spelling,
+which the agent would never look under. `push-images.sh` rejects an
+unrecognised `TARGET_PLATFORM` rather than publishing without a prefix.
+
+The Firework agent resolves the prefix from the architecture of the node it runs
+on, so a node can only ever fetch images built for itself. Host and guest
+architecture must match; before this layout a mismatch surfaced only at microVM
+start, as a guest kernel panic.
+
+A missing image fails at sync with an error naming the key — but only on a node
+that has no local copy. The agent falls back to a cached image whenever an
+object is absent, logging at debug level, so a node that already holds the image
+stays quiet. Verify bucket contents directly rather than waiting for nodes to
+report a gap.
+
+This also means a mixed-architecture fleet needs no extra configuration: node
+configs carry no architecture, so one desired state serves both.
+
+`push-images.sh` takes an explicit `s3` or `gcs` backend argument rather than
+inferring one from whichever bucket is set, and `make push-s3` / `make push-gcs`
+pass it along with `TARGET_PLATFORM`. Each workflow upload step now exports only
+its own provider's bucket, so inference would happen to work — but a local run
+or a future step that exports both must not be able to publish to the wrong
+object store. Inference is therefore still accepted only when exactly one bucket
+variable is set, and errors when both are.
 `scripts/test-push-images.sh` covers this and runs in CI.
 
-Host and guest architecture must match, and a mismatch fails at microVM start
-rather than at deploy time. The AWS data plane in
-`firework-deployment-example` now defaults to x86_64 nodes using nested
-virtualization rather than bare-metal Graviton, so it consumes the amd64 rootfs
-images; the GCP data plane has always been x86_64.
-
-`S3_IMAGES_BUCKET` previously meant the arm64 S3 bucket, so an existing
-deployment that keeps its value will now receive amd64 images in that same
-bucket, replacing the arm64 objects under identical names. That is intended for
-the default x86_64 AWS node. A deployment that still runs Graviton nodes
-(`node_ami_architecture = "arm64"`) must set `S3_IMAGES_BUCKET_ARM64` and point
-`s3_images_bucket_id` at that bucket instead.
+### Objects from the previous layout
+
+Before this layout, objects sat at the bucket root and both the agent and node
+bootstrap read them there. Nothing reads the bucket root now, so any root-level
+`*-rootfs.ext4` left over from that layout is inert and can be deleted whenever
+convenient:
+
+```bash
+aws s3api list-objects-v2 --bucket "$BUCKET" --delimiter / \
+  --query 'Contents[].Key' --output text | tr '\t' '\n'
+```
+
+`list-objects-v2` with `--delimiter /` returns only root-level keys, without the
+`PRE <prefix>/` rows that a plain `aws s3 ls` emits — those rows have no object
+name in the column a naive parse would read, which silently produces empty
+entries.
+
+Fresh nodes have no local image cache, so they depend on the prefixes being
+populated before they boot. Run a `force_rebuild` dispatch after any change to
+this pipeline and confirm both prefixes are present before deploying.
 
 ## CI config validation
 

scripts/push-images.sh

@@ -6,12 +6,20 @@
 #            matching bucket variable is required and the other is ignored.
 #            When omitted the backend is inferred from whichever single bucket
 #            variable is set, and having both set is an error rather than a
-#            silent preference — CI exports both, so inference cannot
-#            distinguish "push to S3" from "push to GCS" on its own.
+#            silent preference: inference cannot distinguish "push to S3" from
+#            "push to GCS" on its own, and a caller with both configured would
+#            otherwise publish to whichever one the script happened to prefer.
+#
+# One bucket per cloud holds every architecture. Objects are uploaded under an
+# <arch>/ prefix, and the Firework agent reads the prefix matching the node it
+# runs on. The prefix uses the Go architecture vocabulary (amd64, arm64) that
+# TARGET_PLATFORM already carries — not the AWS x86_64 spelling, which would be
+# a silent 404 on the agent side.
 #
 # Environment:
 #   S3_IMAGES_BUCKET   destination bucket for the s3 backend
 #   GCS_IMAGES_BUCKET  destination bucket for the gcs backend
+#   TARGET_PLATFORM    platform the images were built for; sets the key prefix
 
 set -euo pipefail
 
@@ -21,15 +29,37 @@ cd "$REPO_ROOT"
 
 BACKEND="${1:-}"
 
+# Deliberately no default. `make build-arm64` sets TARGET_PLATFORM inside its
+# own recipe, so a defaulted push would let `make build-arm64 && make push-s3`
+# publish arm64 images under amd64/ and overwrite them in the now-shared bucket.
+# CI always sets it explicitly, so requiring it costs nothing.
+if [ -z "${TARGET_PLATFORM:-}" ]; then
+    echo "ERROR: TARGET_PLATFORM must be set; it selects the <arch>/ key prefix" >&2
+    echo "Pass the platform the images were built for, e.g.:" >&2
+    echo "  TARGET_PLATFORM=linux/arm64 make push-s3" >&2
+    exit 1
+fi
+
+case "$TARGET_PLATFORM" in
+    linux/amd64 | linux/arm64)
+        TARGET_ARCH="${TARGET_PLATFORM##*/}"
+        ;;
+    *)
+        echo "ERROR: unsupported target platform: $TARGET_PLATFORM" >&2
+        echo "Supported platforms: linux/amd64, linux/arm64" >&2
+        exit 1
+        ;;
+esac
+
 push_s3() {
     if [ -z "${S3_IMAGES_BUCKET:-}" ]; then
         echo "ERROR: S3_IMAGES_BUCKET must be set for the s3 backend" >&2
         exit 1
     fi
     for ext4 in *-rootfs.ext4; do
         [ -f "$ext4" ] || continue
-        echo "Uploading $ext4 to s3://${S3_IMAGES_BUCKET}/${ext4}"
-        aws s3 cp "$ext4" "s3://${S3_IMAGES_BUCKET}/${ext4}"
+        echo "Uploading $ext4 to s3://${S3_IMAGES_BUCKET}/${TARGET_ARCH}/${ext4}"
+        aws s3 cp "$ext4" "s3://${S3_IMAGES_BUCKET}/${TARGET_ARCH}/${ext4}"
     done
 }
 
@@ -40,8 +70,8 @@ push_gcs() {
     fi
     for ext4 in *-rootfs.ext4; do
         [ -f "$ext4" ] || continue
-        echo "Uploading $ext4 to gs://${GCS_IMAGES_BUCKET}/${ext4}"
-        gcloud storage cp "$ext4" "gs://${GCS_IMAGES_BUCKET}/${ext4}"
+        echo "Uploading $ext4 to gs://${GCS_IMAGES_BUCKET}/${TARGET_ARCH}/${ext4}"
+        gcloud storage cp "$ext4" "gs://${GCS_IMAGES_BUCKET}/${TARGET_ARCH}/${ext4}"
     done
 }