From 71578aac3c0b21ad7b04317825441742dff24414 Mon Sep 17 00:00:00 2001 From: Artem Nikitin Date: Sat, 8 Aug 2026 12:48:08 +0200 Subject: [PATCH 1/4] Document the images bucket as holding every architecture The images bucket is no longer architecture-specific. Objects are stored under an / key prefix and the agent reads the prefix matching the node it runs on, so a bucket serves every architecture. No functional Terraform change: s3_images_bucket_id and images_bucket_name were already single-bucket variables, and node IAM is already bucket-wide, so key prefixes need no policy change. What changes is the documented contract. Two statements became wrong rather than merely incomplete: switching to Graviton no longer requires repointing s3_images_bucket_id at a separate arm64 bucket, and node_ami_architecture no longer has to be kept in sync with which bucket a node was given. Following the old runbook would now point a node at a bucket holding both architectures under prefixes it would not find. Co-Authored-By: Claude Opus 5 --- terraform/data-plane/aws/README.md | 9 ++++++--- terraform/data-plane/aws/terraform.tfvars.example | 7 ++++--- terraform/data-plane/aws/variables.tf | 4 ++-- terraform/data-plane/gcp/variables.tf | 2 +- terraform/images-infra/gcp/variables.tf | 2 +- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/terraform/data-plane/aws/README.md b/terraform/data-plane/aws/README.md index ded3854..7eeb594 100644 --- a/terraform/data-plane/aws/README.md +++ b/terraform/data-plane/aws/README.md @@ -191,9 +191,12 @@ node_ami_architecture = "arm64" ``` and rebuild the AMI with `architecture = "arm64"` (see -[packer/aws/README.md](../../../packer/aws/README.md)) and point -`s3_images_bucket_id` at the arm64 rootfs bucket. Host and guest architecture -must match; a mismatch fails at microVM start, not at deploy time. +[packer/aws/README.md](../../../packer/aws/README.md)). `s3_images_bucket_id` +stays as it is: the bucket holds every architecture under an `/` key +prefix and the agent reads the prefix matching its own node, so host and guest +architecture cannot diverge. If the arm64 images have not been published yet, +the node fails at image sync naming the missing key, rather than at microVM +start. **Delete or rebuild `packer/aws/manifest.json` when switching architecture.** `use_packer_manifest_ami` defaults to `true`, so a stale manifest left over from diff --git a/terraform/data-plane/aws/terraform.tfvars.example b/terraform/data-plane/aws/terraform.tfvars.example index 0f4a2ae..a1a8501 100644 --- a/terraform/data-plane/aws/terraform.tfvars.example +++ b/terraform/data-plane/aws/terraform.tfvars.example @@ -60,9 +60,10 @@ node_key_name = "my-key-pair" node_instance_type = "c8i.2xlarge" node_nested_virtualization = true -# To use bare-metal Graviton nodes instead, all three must change together, and -# the AMI must be rebuilt with architecture = "arm64" and s3_images_bucket_id -# pointed at the arm64 rootfs bucket: +# To use bare-metal Graviton nodes instead, all three must change together and +# the AMI must be rebuilt with architecture = "arm64". s3_images_bucket_id is +# unchanged: the bucket holds every architecture under an / key prefix and +# the agent reads the one matching its node. # node_instance_type = "c6g.metal" # node_nested_virtualization = false # node_ami_architecture = "arm64" diff --git a/terraform/data-plane/aws/variables.tf b/terraform/data-plane/aws/variables.tf index 08924a9..2c2127d 100644 --- a/terraform/data-plane/aws/variables.tf +++ b/terraform/data-plane/aws/variables.tf @@ -93,7 +93,7 @@ variable "nat_gateway_mode" { # --- S3 (pre-existing, managed outside this stack) --- variable "s3_images_bucket_id" { - description = "Name/ID of the pre-existing S3 images bucket (managed by CI, not Terraform)" + description = "Name/ID of the pre-existing S3 images bucket (managed by CI, not Terraform). Holds every architecture under an / key prefix; nodes read the prefix matching their own architecture." type = string } @@ -148,7 +148,7 @@ variable "node_ami_owners" { } variable "node_ami_architecture" { - description = "Architecture filter used when resolving AMI by name pattern. Must match the architecture the node AMI was built for and the rootfs images in s3_images_bucket_id." + description = "Architecture filter used when resolving AMI by name pattern. Must match the architecture the node AMI was built for. The images bucket holds every architecture, so nodes resolve their rootfs images automatically and this does not need to be matched against a bucket." type = string default = "x86_64" diff --git a/terraform/data-plane/gcp/variables.tf b/terraform/data-plane/gcp/variables.tf index 65be6e5..c688fc6 100644 --- a/terraform/data-plane/gcp/variables.tf +++ b/terraform/data-plane/gcp/variables.tf @@ -226,7 +226,7 @@ variable "config_prefix" { variable "images_bucket_name" { type = string - description = "Globally unique amd64 images bucket name (created by the images-infra stack)" + description = "Globally unique images bucket name (created by the images-infra stack). Holds every architecture under an / key prefix." } variable "registry_url" { diff --git a/terraform/images-infra/gcp/variables.tf b/terraform/images-infra/gcp/variables.tf index c8e3e89..cf60617 100644 --- a/terraform/images-infra/gcp/variables.tf +++ b/terraform/images-infra/gcp/variables.tf @@ -11,7 +11,7 @@ variable "gcp_region" { variable "images_bucket_name" { type = string - description = "Globally unique name for the amd64 node images bucket. Used by Packer to upload images and by data-plane nodes to rsync images at boot." + description = "Globally unique name for the node images bucket. Holds every architecture under an / key prefix; nodes read the prefix matching their own architecture. Used by Packer to upload images and by data-plane nodes to rsync images at boot." } variable "packer_service_account" { From 27e3b32ef63f1d3973bc6d58c2bcfcc18960be2c Mon Sep 17 00:00:00 2001 From: Artem Nikitin Date: Sat, 8 Aug 2026 17:01:37 +0200 Subject: [PATCH 2/4] Sweep the remaining architecture-specific bucket prose The IAM policy docs and the GCP tfvars example still described a bucket per architecture. The JSON policies themselves needed no change: both clouds grant bucket-wide object access with no prefix conditions, so key prefixes are already covered. AGENTS.md asks for iam-policies/ to be kept current with changes, and these were the last places still telling a reader to provision or select an architecture-specific bucket. The AWS AMI row still says ARM64: that one is genuinely architecture-specific, since it describes building the Graviton node image rather than uploading rootfs images. Co-Authored-By: Claude Opus 5 --- iam-policies/aws/README.md | 2 +- iam-policies/gcp/01-ci-images-gcs.md | 8 +++++--- terraform/data-plane/gcp/terraform.tfvars.example | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/iam-policies/aws/README.md b/iam-policies/aws/README.md index d3be00e..37204d2 100644 --- a/iam-policies/aws/README.md +++ b/iam-policies/aws/README.md @@ -4,7 +4,7 @@ Replace `IMAGES_BUCKET_NAME` and `ACCOUNT_ID` in the JSON examples before use. | File | Principal | Purpose | |---|---|---| -| `01-ci-images-s3.json` | GitHub Actions CI | Upload ARM64 rootfs images to S3 | +| `01-ci-images-s3.json` | GitHub Actions CI | Upload rootfs images for every architecture to S3 | | `02-terraform-deploy.json` | Terraform operator | Deploy AWS control and data planes | | `03-packer-build.json` | Packer operator | Build the ARM64 AMI | diff --git a/iam-policies/gcp/01-ci-images-gcs.md b/iam-policies/gcp/01-ci-images-gcs.md index fe5c7e2..5947a7e 100644 --- a/iam-policies/gcp/01-ci-images-gcs.md +++ b/iam-policies/gcp/01-ci-images-gcs.md @@ -1,7 +1,9 @@ # CI image uploader Grant the GitHub Actions service account `roles/storage.objectAdmin` on the -amd64 images bucket only. Do not grant the role at project level. +images bucket only. Do not grant the role at project level. One bucket holds +every architecture under an `/` key prefix, so a single bucket-scoped +binding covers all of them. Use Workload Identity Federation and grant the repository principal `roles/iam.workloadIdentityUser` on that service account. No JSON service @@ -12,7 +14,7 @@ account key is required. Bucket-level upload access (note: bucket-scoped, not project-scoped): ```bash -gcloud storage buckets add-iam-policy-binding gs://AMD64_IMAGES_BUCKET \ +gcloud storage buckets add-iam-policy-binding gs://IMAGES_BUCKET \ --member="serviceAccount:CI_SA@PROJECT_ID.iam.gserviceaccount.com" \ --role="roles/storage.objectAdmin" ``` @@ -28,7 +30,7 @@ gcloud iam service-accounts add-iam-policy-binding \ ## Granting via Console -- Bucket access: **Cloud Storage → Buckets →** select the amd64 images bucket +- Bucket access: **Cloud Storage → Buckets →** select the images bucket **→ Permissions → Grant access**, add the CI service account, role **Storage Object Admin**. - WIF binding: **IAM & Admin → Service Accounts →** select the CI service diff --git a/terraform/data-plane/gcp/terraform.tfvars.example b/terraform/data-plane/gcp/terraform.tfvars.example index 2597e4d..acf9f2c 100644 --- a/terraform/data-plane/gcp/terraform.tfvars.example +++ b/terraform/data-plane/gcp/terraform.tfvars.example @@ -8,7 +8,7 @@ dns_zone_name = "firework-gcp" # Single source of truth for wildcard DNS, the wildcard TLS certificate, and the # agent ingress_domain. Services route as .. base_domain = "gcp.example.com" -images_bucket_name = "firework-images-amd64-my-gcp-project" +images_bucket_name = "firework-images-my-gcp-project" # Default false: Private Google Access and registry peering are enough for the # demo. Enable only when guest workloads or node debugging need public egress. From 4de27ea2ded88dea4cd2d042100e87eef19e2ff1 Mon Sep 17 00:00:00 2001 From: Artem Nikitin Date: Sun, 9 Aug 2026 12:26:17 +0200 Subject: [PATCH 3/4] Select the node's architecture prefix during bootstrap Node bootstrap prefetches rootfs images before the agent starts, and both providers assumed objects sat at the bucket root. With images stored under an / prefix that assumption breaks a fresh deploy, not just an upgrade. On AWS, `aws s3 sync` preserves key structure and its `*` matches across `/`, so a node downloaded every architecture into nested directories the agent never reads, then the agent fetched its own copies on top. For the current tenant image set that is roughly 36 GiB of prefetch plus 18 GiB of agent cache against a 50 GB root volume. On GCP the opposite: `gcloud storage`'s `*` does not cross `/`, so gs://bucket/*.ext4 matched nothing. Under `set -euo pipefail` the failure aborted the startup script before the agent config was written, leaving a node that never enrolled. Both now derive the prefix from `uname -m` and flatten it into /var/lib/images, matching where the agent resolves images. The GCP copy is additionally best-effort: prefetch is an optimization, and the agent syncs images itself, so an empty prefix must not prevent a node from starting. Co-Authored-By: Claude Opus 5 --- terraform/data-plane/aws/README.md | 4 ++-- .../data-plane/aws/templates/user-data.sh.tpl | 20 ++++++++++++++-- .../gcp/templates/startup-script.sh.tpl | 23 ++++++++++++++++++- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/terraform/data-plane/aws/README.md b/terraform/data-plane/aws/README.md index 7eeb594..5c62363 100644 --- a/terraform/data-plane/aws/README.md +++ b/terraform/data-plane/aws/README.md @@ -193,8 +193,8 @@ node_ami_architecture = "arm64" and rebuild the AMI with `architecture = "arm64"` (see [packer/aws/README.md](../../../packer/aws/README.md)). `s3_images_bucket_id` stays as it is: the bucket holds every architecture under an `/` key -prefix and the agent reads the prefix matching its own node, so host and guest -architecture cannot diverge. If the arm64 images have not been published yet, +prefix, and both the node bootstrap and the agent select the prefix from +`uname -m`, so host and guest architecture cannot diverge. If the arm64 images have not been published yet, the node fails at image sync naming the missing key, rather than at microVM start. diff --git a/terraform/data-plane/aws/templates/user-data.sh.tpl b/terraform/data-plane/aws/templates/user-data.sh.tpl index 154cb90..30408b3 100644 --- a/terraform/data-plane/aws/templates/user-data.sh.tpl +++ b/terraform/data-plane/aws/templates/user-data.sh.tpl @@ -317,9 +317,25 @@ if [ -x /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl ]; then fi # --- 1. Download rootfs images from S3 --- -echo "==> Downloading rootfs images from s3://$S3_IMAGES_BUCKET/" +# The bucket holds every architecture under an / key prefix, so sync only +# this node's prefix and flatten it: the agent resolves images at +# $IMAGES_DIR/, not $IMAGES_DIR//. +# +# Syncing the bucket root instead would download every architecture into nested +# directories the agent never reads, and on a multi-tenant image set that alone +# can exceed the root volume before the agent fetches the copies it does use. +case "$(uname -m)" in + x86_64) IMAGES_ARCH="amd64" ;; + aarch64) IMAGES_ARCH="arm64" ;; + *) + echo "ERROR: unsupported machine architecture: $(uname -m)" >&2 + exit 1 + ;; +esac + +echo "==> Downloading $IMAGES_ARCH rootfs images from s3://$S3_IMAGES_BUCKET/$IMAGES_ARCH/" retry "download rootfs images" 20 5 \ - aws s3 sync "s3://$S3_IMAGES_BUCKET/" "$IMAGES_DIR/" \ + aws s3 sync "s3://$S3_IMAGES_BUCKET/$IMAGES_ARCH/" "$IMAGES_DIR/" \ --region "$S3_REGION" \ --exclude "*" --include "*.ext4" echo "==> Images downloaded" diff --git a/terraform/data-plane/gcp/templates/startup-script.sh.tpl b/terraform/data-plane/gcp/templates/startup-script.sh.tpl index 7e621d0..5114647 100644 --- a/terraform/data-plane/gcp/templates/startup-script.sh.tpl +++ b/terraform/data-plane/gcp/templates/startup-script.sh.tpl @@ -96,7 +96,28 @@ fi read_secret "${registry_ca_secret}" > /etc/firework/pki/node-ca.crt REGISTRY_BOOTSTRAP_TOKEN=$(read_secret "${registry_bootstrap_secret}") -gcloud storage cp --no-clobber "gs://${gcs_images_bucket}/*.ext4" /var/lib/images/ +# The bucket holds every architecture under an / key prefix, so copy only +# this node's prefix and flatten it: the agent resolves images at +# /var/lib/images/, not /var/lib/images//. A bare +# gs://bucket/*.ext4 matches nothing here, because gcloud's * does not cross a +# path separator. +case "$(uname -m)" in + x86_64) IMAGES_ARCH="amd64" ;; + aarch64) IMAGES_ARCH="arm64" ;; + *) + echo "ERROR: unsupported machine architecture: $(uname -m)" >&2 + exit 1 + ;; +esac + +# Best-effort prefetch: the agent syncs images itself, so an empty or +# not-yet-populated prefix must not abort startup before the agent config is +# written. Without the guard, set -e turns "no URLs matched" into a node that +# never enrolls. +if ! gcloud storage cp --no-clobber \ + "gs://${gcs_images_bucket}/$IMAGES_ARCH/*.ext4" /var/lib/images/; then + echo "WARNING: no $IMAGES_ARCH images prefetched from gs://${gcs_images_bucket}/$IMAGES_ARCH/; the agent will sync them" >&2 +fi cat > /etc/firework/agent.yaml < Date: Sun, 9 Aug 2026 12:40:41 +0200 Subject: [PATCH 4/4] Stop prefetching rootfs images during node bootstrap Neither cloud CLI writes Firework's write-token sidecars, so every prefetched image was downloaded a second time by the agent moments after it started. Bootstrap also pulled every image for the node's architecture, while the agent downloads only the images actually assigned to that node. The agent is simply the better place for this: it fetches the assigned subset and maintains the sidecars that let a restart skip unchanged objects. Both bucket variables are required by the data-plane stacks and are always written into the agent config, so image sync is never disabled and removing the prefetch cannot strand a node. This also retires the GCP startup hazard entirely rather than guarding it: there is no longer a copy that can match nothing and abort the script under `set -e` before the agent config is written. The architecture guidance is corrected alongside it. Three statements in the AWS data-plane guide disagreed with each other about whether switching to Graviton means repointing the images bucket, and the Packer guide still warned about aiming a node at an arm64 rootfs bucket. Under one bucket only the AMI, instance type, and architecture setting change. The claim that bootstrap and the agent both select via `uname -m` was also wrong in a way this commit makes moot: bootstrap no longer selects anything, and the agent uses its own binary's architecture. Co-Authored-By: Claude Opus 5 --- packer/aws/README.md | 6 ++-- terraform/data-plane/aws/README.md | 15 +++++---- .../data-plane/aws/templates/user-data.sh.tpl | 29 ++++------------- .../gcp/templates/startup-script.sh.tpl | 32 ++++++------------- 4 files changed, 29 insertions(+), 53 deletions(-) diff --git a/packer/aws/README.md b/packer/aws/README.md index e02711d..dc807fb 100644 --- a/packer/aws/README.md +++ b/packer/aws/README.md @@ -18,8 +18,10 @@ The architecture must be consistent across three places: | `node_ami_architecture` and `node_instance_type` | data-plane stack | | rootfs image set | S3 images bucket (`firework-gitops-example` builds both) | -Host and guest architecture must match, so pointing an x86_64 node at an arm64 -rootfs bucket fails at microVM start, not at deploy time. +Host and guest architecture must match. One bucket holds both, and the agent +resolves the `/` prefix from its own architecture, so a node cannot be +pointed at the wrong rootfs set. An architecture that has not been published yet +fails at image sync naming the missing key. All provisioning scripts in `packer/scripts/` resolve their downloads from `uname -m`, so they need no changes between architectures. diff --git a/terraform/data-plane/aws/README.md b/terraform/data-plane/aws/README.md index 5c62363..9c6a73e 100644 --- a/terraform/data-plane/aws/README.md +++ b/terraform/data-plane/aws/README.md @@ -182,7 +182,7 @@ performance sensitive or have strict latency requirements, so **do not use this stack to benchmark microVM boot latency.** To go back to bare-metal Graviton nodes, three settings must change together, -plus the AMI and the image bucket: +plus the AMI: ```hcl node_instance_type = "c6g.metal" @@ -193,10 +193,10 @@ node_ami_architecture = "arm64" and rebuild the AMI with `architecture = "arm64"` (see [packer/aws/README.md](../../../packer/aws/README.md)). `s3_images_bucket_id` stays as it is: the bucket holds every architecture under an `/` key -prefix, and both the node bootstrap and the agent select the prefix from -`uname -m`, so host and guest architecture cannot diverge. If the arm64 images have not been published yet, -the node fails at image sync naming the missing key, rather than at microVM -start. +prefix, and the agent resolves the prefix from its own binary's architecture, so +host and guest architecture cannot diverge. If the arm64 images have not been +published yet, the node fails at image sync naming the missing key, rather than +at microVM start. **Delete or rebuild `packer/aws/manifest.json` when switching architecture.** `use_packer_manifest_ami` defaults to `true`, so a stale manifest left over from @@ -242,8 +242,9 @@ node_ami_architecture = "arm64" That reproduces the previous topology exactly, so the upgrade is a no-op. Change them one at a time when you are ready, and expect nodes to be replaced. -Switching architecture additionally requires rebuilding the AMI and repointing -`s3_images_bucket_id` at a matching rootfs image set, as described above. +Switching architecture additionally requires rebuilding the AMI, as described +above. `s3_images_bucket_id` does not change: one bucket holds every +architecture. #### Subnet keys changed from index to availability zone diff --git a/terraform/data-plane/aws/templates/user-data.sh.tpl b/terraform/data-plane/aws/templates/user-data.sh.tpl index 30408b3..367b3f2 100644 --- a/terraform/data-plane/aws/templates/user-data.sh.tpl +++ b/terraform/data-plane/aws/templates/user-data.sh.tpl @@ -316,29 +316,14 @@ if [ -x /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl ]; then -s || true fi -# --- 1. Download rootfs images from S3 --- -# The bucket holds every architecture under an / key prefix, so sync only -# this node's prefix and flatten it: the agent resolves images at -# $IMAGES_DIR/, not $IMAGES_DIR//. +# --- 1. Rootfs images --- +# Not prefetched here. The agent syncs images itself from s3_images_bucket, and +# it is the better place to do it: it downloads only the images assigned to this +# node rather than every image for this architecture, and it maintains the +# write-token sidecars that let a restart skip unchanged objects. # -# Syncing the bucket root instead would download every architecture into nested -# directories the agent never reads, and on a multi-tenant image set that alone -# can exceed the root volume before the agent fetches the copies it does use. -case "$(uname -m)" in - x86_64) IMAGES_ARCH="amd64" ;; - aarch64) IMAGES_ARCH="arm64" ;; - *) - echo "ERROR: unsupported machine architecture: $(uname -m)" >&2 - exit 1 - ;; -esac - -echo "==> Downloading $IMAGES_ARCH rootfs images from s3://$S3_IMAGES_BUCKET/$IMAGES_ARCH/" -retry "download rootfs images" 20 5 \ - aws s3 sync "s3://$S3_IMAGES_BUCKET/$IMAGES_ARCH/" "$IMAGES_DIR/" \ - --region "$S3_REGION" \ - --exclude "*" --include "*.ext4" -echo "==> Images downloaded" +# A prefetch here cannot write those sidecars, so every prefetched image was +# downloaded a second time by the agent moments later. # --- 2. Write agent config --- # Detect the primary network interface for masquerade (agent needs this). diff --git a/terraform/data-plane/gcp/templates/startup-script.sh.tpl b/terraform/data-plane/gcp/templates/startup-script.sh.tpl index 5114647..a76988e 100644 --- a/terraform/data-plane/gcp/templates/startup-script.sh.tpl +++ b/terraform/data-plane/gcp/templates/startup-script.sh.tpl @@ -96,28 +96,16 @@ fi read_secret "${registry_ca_secret}" > /etc/firework/pki/node-ca.crt REGISTRY_BOOTSTRAP_TOKEN=$(read_secret "${registry_bootstrap_secret}") -# The bucket holds every architecture under an / key prefix, so copy only -# this node's prefix and flatten it: the agent resolves images at -# /var/lib/images/, not /var/lib/images//. A bare -# gs://bucket/*.ext4 matches nothing here, because gcloud's * does not cross a -# path separator. -case "$(uname -m)" in - x86_64) IMAGES_ARCH="amd64" ;; - aarch64) IMAGES_ARCH="arm64" ;; - *) - echo "ERROR: unsupported machine architecture: $(uname -m)" >&2 - exit 1 - ;; -esac - -# Best-effort prefetch: the agent syncs images itself, so an empty or -# not-yet-populated prefix must not abort startup before the agent config is -# written. Without the guard, set -e turns "no URLs matched" into a node that -# never enrolls. -if ! gcloud storage cp --no-clobber \ - "gs://${gcs_images_bucket}/$IMAGES_ARCH/*.ext4" /var/lib/images/; then - echo "WARNING: no $IMAGES_ARCH images prefetched from gs://${gcs_images_bucket}/$IMAGES_ARCH/; the agent will sync them" >&2 -fi +# Rootfs images are not prefetched here. The agent syncs them itself from +# gcs_images_bucket, and it is the better place to do it: it downloads only the +# images assigned to this node rather than every image for this architecture, +# and it maintains the write-token sidecars that let a restart skip unchanged +# objects. +# +# A prefetch here cannot write those sidecars, so every prefetched image was +# downloaded a second time by the agent moments later. It was also a startup +# hazard: under `set -e`, a copy that matched nothing aborted this script before +# the agent config below was written, leaving a node that never enrolled. cat > /etc/firework/agent.yaml <