From 2cb2ea2d441ec3f4aa598f303136b282551ce27c Mon Sep 17 00:00:00 2001
From: Artem Nikitin <artemnikitin@artemnikitin.com>
Date: Wed, 5 Aug 2026 16:45:46 +0200
Subject: [PATCH 1/2] Initial commit

---
 Makefile                                      |   8 +-
 validation/e2e-local/README.md                |  67 +++
 validation/e2e-local/scripts/check-env.sh     |  41 ++
 .../e2e-local/scripts/collect-diagnostics.sh  |  53 ++
 validation/e2e-local/scripts/destroy-lab.sh   |  38 ++
 validation/e2e-local/scripts/generate-pki.sh  |  38 ++
 validation/e2e-local/scripts/run-guest.sh     | 463 ++++++++++++++++++
 validation/e2e-local/scripts/validate.sh      | 185 +++++++
 8 files changed, 892 insertions(+), 1 deletion(-)
 create mode 100644 validation/e2e-local/README.md
 create mode 100755 validation/e2e-local/scripts/check-env.sh
 create mode 100755 validation/e2e-local/scripts/collect-diagnostics.sh
 create mode 100755 validation/e2e-local/scripts/destroy-lab.sh
 create mode 100755 validation/e2e-local/scripts/generate-pki.sh
 create mode 100755 validation/e2e-local/scripts/run-guest.sh
 create mode 100755 validation/e2e-local/scripts/validate.sh

diff --git a/Makefile b/Makefile
index 7486b6c..78db0b2 100644
--- a/Makefile
+++ b/Makefile
@@ -26,7 +26,7 @@ LDFLAGS   := -s -w \
 	-X '$(MODULE)/internal/version.Commit=$(COMMIT)' \
 	-X '$(MODULE)/internal/version.BuildTime=$(BUILD_TIME)'
 
-.PHONY: all build-all build-agent build-controlplane build-configcheck build-fireworkctl build-fireworkctl-release build-fc-init build-linux-amd64 build-linux-arm64 clean test test-verbose test-race lint vet fmt tidy run smoke-local docker-build-controlplane-image docker-push-controlplane-image push-controlplane-image install help
+.PHONY: all build-all build-agent build-controlplane build-configcheck build-fireworkctl build-fireworkctl-release build-fc-init build-linux-amd64 build-linux-arm64 clean test test-verbose test-race lint vet fmt tidy run smoke-local validate-e2e-local validate-e2e-local-clean docker-build-controlplane-image docker-push-controlplane-image push-controlplane-image install help
 
 all: build-all ## Alias for build-all
 
@@ -118,6 +118,12 @@ run: build-agent ## Build and run with example config
 smoke-local: ## Run local smoke test with fake firecracker
 	./scripts/smoke-local.sh
 
+validate-e2e-local: ## Run the local two-node E2E lab with real Firecracker and S3
+	./validation/e2e-local/scripts/validate.sh
+
+validate-e2e-local-clean: ## Destroy a retained local E2E lab (set FIREWORK_E2E_MANIFEST)
+	./validation/e2e-local/scripts/destroy-lab.sh "$(FIREWORK_E2E_MANIFEST)"
+
 docker-build-controlplane-image: ## Build control-plane image locally (linux/amd64)
 	docker buildx build --platform linux/amd64 --file Dockerfile.controlplane \
 		--build-arg VERSION="$(VERSION)" \
diff --git a/validation/e2e-local/README.md b/validation/e2e-local/README.md
new file mode 100644
index 0000000..c538b1d
--- /dev/null
+++ b/validation/e2e-local/README.md
@@ -0,0 +1,67 @@
+# Local two-node E2E validation
+
+This harness runs one complete local Firework setup:
+
+```text
+one control-plane process (registry + events + controller + API)
+                         |
+                 real AWS S3 bucket
+                         |
+          node-a network namespace -- node-b network namespace
+             firework-agent             firework-agent
+                  |                            |
+             Firecracker VM               Firecracker VM
+```
+
+The control plane reconciles a disposable local Git repository. It publishes
+the desired state and rendered node configs into a unique real S3 bucket. Both
+agents enroll over mTLS, consume their rendered configs from that bucket, and
+run one Firecracker VM each. The caller VM is deliberately placed on the
+opposite node from the responder VM and reaches it through a rendered
+cross-node link.
+
+This is local developer validation for now. CI and the unprivileged-build /
+privileged-lab handoff are the next milestone.
+
+## Prerequisites
+
+- Linux with root, `/dev/kvm`, `/dev/net/tun`, `ip`, and `iptables`; or
+  Apple Silicon macOS with Lima, nested virtualization, and a Linux guest
+  that exposes `/dev/kvm`.
+- Go and the repository build toolchain.
+- AWS credentials with permission to create, list, write, and delete a
+  disposable S3 bucket. The runner creates and deletes its own bucket on every
+  run; it refuses to reuse an existing bucket name.
+- A Linux Firecracker binary, an uncompressed Linux kernel, and an ext4
+  rootfs containing BusyBox. The runner copies `fc-init` and the disposable
+  E2E init script into a temporary copy of the rootfs, so the supplied rootfs
+  is not modified.
+
+Set these paths before running:
+
+```bash
+export FIREWORK_E2E_FIRECRACKER_BIN=/path/to/firecracker
+export FIREWORK_E2E_KERNEL=/path/to/vmlinux
+export FIREWORK_E2E_ROOTFS=/path/to/rootfs.ext4
+export AWS_REGION=us-east-1
+make validate-e2e-local
+```
+
+When `AWS_PROFILE` is used instead of environment credentials, the host
+runner exports that profile's current credentials before entering Lima. This
+also supports short-lived credentials obtained through the AWS CLI.
+
+Useful local-only options:
+
+- `FIREWORK_E2E_MODE=linux` forces native Linux execution.
+- `FIREWORK_E2E_MODE=lima` forces the macOS Lima adapter.
+- `FIREWORK_E2E_KEEP=1` retains the lab and prints a manifest path and cleanup
+  command. Use `make validate-e2e-local-clean FIREWORK_E2E_MANIFEST=...` after
+  inspection.
+- `FIREWORK_E2E_TIMEOUT=600` changes the bounded scenario timeout.
+
+The runner writes logs, redacted configuration diagnostics, S3 object
+inventory, network state, and a run manifest under a temporary directory. AWS
+credentials are passed to the runtime through the process environment and are
+never written to the manifest or generated configuration files; the generated
+files do contain short-lived local mTLS/bootstrap tokens needed by the lab.
diff --git a/validation/e2e-local/scripts/check-env.sh b/validation/e2e-local/scripts/check-env.sh
new file mode 100755
index 0000000..7be8255
--- /dev/null
+++ b/validation/e2e-local/scripts/check-env.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+die() {
+  printf 'ERROR: %s\n' "$*" >&2
+  exit 1
+}
+
+require_cmd() {
+  command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
+}
+
+require_cmd aws
+require_cmd curl
+require_cmd git
+require_cmd jq
+require_cmd openssl
+
+[[ -n "${AWS_REGION:-${AWS_DEFAULT_REGION:-}}" ]] || die "AWS_REGION or AWS_DEFAULT_REGION is required"
+
+[[ -n "${FIREWORK_E2E_FIRECRACKER_BIN:-}" ]] || die "FIREWORK_E2E_FIRECRACKER_BIN is required"
+[[ -x "${FIREWORK_E2E_FIRECRACKER_BIN}" ]] || die "Firecracker binary is not executable: ${FIREWORK_E2E_FIRECRACKER_BIN}"
+[[ -n "${FIREWORK_E2E_KERNEL:-}" ]] || die "FIREWORK_E2E_KERNEL is required"
+[[ -r "${FIREWORK_E2E_KERNEL}" ]] || die "kernel is not readable: ${FIREWORK_E2E_KERNEL}"
+[[ -n "${FIREWORK_E2E_ROOTFS:-}" ]] || die "FIREWORK_E2E_ROOTFS is required"
+[[ -r "${FIREWORK_E2E_ROOTFS}" ]] || die "rootfs is not readable: ${FIREWORK_E2E_ROOTFS}"
+
+case "$(uname -s)" in
+  Darwin)
+    require_cmd limactl
+    ;;
+  Linux)
+    require_cmd ip
+    require_cmd iptables
+    [[ -r /dev/kvm && -w /dev/kvm ]] || die "/dev/kvm is not readable and writable"
+    [[ -e /dev/net/tun ]] || die "/dev/net/tun is required"
+    ;;
+  *)
+    die "unsupported host OS: $(uname -s)"
+    ;;
+esac
diff --git a/validation/e2e-local/scripts/collect-diagnostics.sh b/validation/e2e-local/scripts/collect-diagnostics.sh
new file mode 100755
index 0000000..5e8922d
--- /dev/null
+++ b/validation/e2e-local/scripts/collect-diagnostics.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -u
+
+RUN_DIR=${1:?usage: collect-diagnostics.sh RUN_DIR}
+mkdir -p "$RUN_DIR/diagnostics"
+
+out="$RUN_DIR/diagnostics"
+
+for config in "$RUN_DIR"/config/*.yaml; do
+  [[ -f "$config" ]] || continue
+  name=$(basename "$config")
+  sed -E \
+    -e 's/^([[:space:]]*(operator_token|registry_bootstrap_token|github_webhook_secret):).*/\1 "<redacted>"/' \
+    -e 's/^([[:space:]]*-[[:space:]]*token:).*/\1 "<redacted>"/' \
+    "$config" > "$out/$name"
+done
+
+ip address show > "$out/ip-address.txt" 2>&1 || true
+ip route show table all > "$out/ip-routes.txt" 2>&1 || true
+iptables-save > "$out/iptables.txt" 2>&1 || true
+
+for namespace in fw-e2e-node-a fw-e2e-node-b; do
+  ip netns exec "$namespace" ip address show > "$out/$namespace-ip-address.txt" 2>&1 || true
+  ip netns exec "$namespace" ip route show table all > "$out/$namespace-ip-routes.txt" 2>&1 || true
+  ip netns exec "$namespace" iptables-save > "$out/$namespace-iptables.txt" 2>&1 || true
+done
+
+if [[ -n "${CONTROLPLANE_PID:-}" ]]; then
+  ps -o pid,ppid,state,etime,args -p "$CONTROLPLANE_PID" > "$out/controlplane-process.txt" 2>&1 || true
+fi
+for pid in ${AGENT_PIDS:-}; do
+  ps -o pid,ppid,state,etime,args -p "$pid" >> "$out/agent-processes.txt" 2>&1 || true
+done
+
+if [[ -n "${CONTROLPLANE_CURL_URL:-}" && -n "${CONTROLPLANE_CA_FILE:-}" ]]; then
+  curl --silent --show-error --cacert "$CONTROLPLANE_CA_FILE" \
+    -H "Authorization: Bearer ${CONTROLPLANE_OPERATOR_TOKEN:-}" \
+    "$CONTROLPLANE_CURL_URL/v1/nodes" > "$out/controlplane-nodes.json" 2>&1 || true
+  curl --silent --show-error --cacert "$CONTROLPLANE_CA_FILE" \
+    -H "Authorization: Bearer ${CONTROLPLANE_OPERATOR_TOKEN:-}" \
+    "$CONTROLPLANE_CURL_URL/v1/services" > "$out/controlplane-services.json" 2>&1 || true
+fi
+
+for endpoint in ${AGENT_ENDPOINTS:-}; do
+  name=${endpoint%%=*}
+  url=${endpoint#*=}
+  curl --silent --show-error --fail "$url/status" > "$out/${name}-status.json" 2>&1 || true
+done
+
+if [[ -n "${E2E_BUCKET:-}" ]]; then
+  aws s3api list-objects-v2 --bucket "$E2E_BUCKET" --prefix cp/v1/ \
+    --output json > "$out/s3-cp-v1-inventory.json" 2>&1 || true
+fi
diff --git a/validation/e2e-local/scripts/destroy-lab.sh b/validation/e2e-local/scripts/destroy-lab.sh
new file mode 100755
index 0000000..60b30ef
--- /dev/null
+++ b/validation/e2e-local/scripts/destroy-lab.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+manifest=${1:-}
+if [[ -z "$manifest" ]]; then
+  printf 'usage: make validate-e2e-local-clean FIREWORK_E2E_MANIFEST=/path/to/manifest.json\n' >&2
+  exit 2
+fi
+
+command -v jq >/dev/null 2>&1 || { printf 'ERROR: jq is required\n' >&2; exit 1; }
+command -v aws >/dev/null 2>&1 || { printf 'ERROR: aws is required\n' >&2; exit 1; }
+
+bucket=$(jq -r '.bucket // empty' "$manifest")
+region=$(jq -r '.region // empty' "$manifest")
+mode=$(jq -r '.mode // empty' "$manifest")
+instance=$(jq -r '.lima_instance // empty' "$manifest")
+
+[[ -n "$bucket" && -n "$region" ]] || { printf 'ERROR: invalid manifest: %s\n' "$manifest" >&2; exit 1; }
+export AWS_REGION="$region"
+export AWS_DEFAULT_REGION="$region"
+export AWS_EC2_METADATA_DISABLED=true
+
+if [[ "$mode" == lima && -n "$instance" ]] && command -v limactl >/dev/null 2>&1; then
+  limactl stop --force "$instance" >/dev/null 2>&1 || true
+  limactl delete --force "$instance" >/dev/null 2>&1 || true
+elif [[ "$mode" == linux ]]; then
+  for pid in $(jq -r '(.agent_pids[]?, .controlplane_pid?) | select(type == "number")' "$manifest"); do
+    sudo kill -TERM "$pid" >/dev/null 2>&1 || true
+  done
+  for namespace in fw-e2e-node-a fw-e2e-node-b; do
+    sudo ip netns del "$namespace" >/dev/null 2>&1 || true
+  done
+  sudo ip link del fw-e2e-br >/dev/null 2>&1 || true
+fi
+
+aws s3 rm "s3://$bucket" --recursive >/dev/null
+aws s3api delete-bucket --bucket "$bucket" >/dev/null
+printf 'destroyed local E2E lab from %s\n' "$manifest"
diff --git a/validation/e2e-local/scripts/generate-pki.sh b/validation/e2e-local/scripts/generate-pki.sh
new file mode 100755
index 0000000..142e3f3
--- /dev/null
+++ b/validation/e2e-local/scripts/generate-pki.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+OUT_DIR=${1:?usage: generate-pki.sh OUT_DIR}
+mkdir -p "$OUT_DIR"
+umask 077
+
+openssl req -x509 -newkey rsa:2048 -nodes \
+  -keyout "$OUT_DIR/ca.key" \
+  -out "$OUT_DIR/ca.crt" \
+  -days 2 \
+  -subj "/CN=firework-local-e2e-ca" \
+  >/dev/null 2>&1
+
+openssl req -newkey rsa:2048 -nodes \
+  -keyout "$OUT_DIR/controlplane.key" \
+  -out "$OUT_DIR/controlplane.csr" \
+  -subj "/CN=controlplane.local" \
+  >/dev/null 2>&1
+
+cat > "$OUT_DIR/controlplane.ext" <<'EOF'
+basicConstraints=CA:FALSE
+keyUsage=digitalSignature,keyEncipherment
+extendedKeyUsage=serverAuth
+subjectAltName=DNS:controlplane.local,IP:127.0.0.1,IP:10.254.0.1
+EOF
+
+openssl x509 -req \
+  -in "$OUT_DIR/controlplane.csr" \
+  -CA "$OUT_DIR/ca.crt" \
+  -CAkey "$OUT_DIR/ca.key" \
+  -CAcreateserial \
+  -out "$OUT_DIR/controlplane.crt" \
+  -days 2 \
+  -extfile "$OUT_DIR/controlplane.ext" \
+  >/dev/null 2>&1
+
+rm -f "$OUT_DIR/controlplane.csr" "$OUT_DIR/controlplane.ext" "$OUT_DIR/ca.srl"
diff --git a/validation/e2e-local/scripts/run-guest.sh b/validation/e2e-local/scripts/run-guest.sh
new file mode 100755
index 0000000..5c88e84
--- /dev/null
+++ b/validation/e2e-local/scripts/run-guest.sh
@@ -0,0 +1,463 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+RUN_DIR=${1:?usage: run-guest.sh RUN_DIR BUCKET REGION COMMIT MODE}
+E2E_BUCKET=${2:?missing bucket}
+AWS_REGION_VALUE=${3:?missing AWS region}
+FIREWORK_COMMIT=${4:?missing Firework commit}
+RUN_MODE=${5:?missing run mode}
+
+export AWS_REGION="$AWS_REGION_VALUE"
+export AWS_DEFAULT_REGION="$AWS_REGION_VALUE"
+export AWS_EC2_METADATA_DISABLED=true
+
+SCRIPT_DIR="${FIREWORK_E2E_SCRIPT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
+BIN_DIR="$RUN_DIR/bin"
+IMAGE_DIR="$RUN_DIR/images"
+CONFIG_DIR="$RUN_DIR/config"
+LOG_DIR="$RUN_DIR/logs"
+PKI_DIR="$RUN_DIR/pki"
+STATE_DIR="$RUN_DIR/state"
+ROOTFS_MOUNT="$RUN_DIR/rootfs-mount"
+GUEST_ROOTFS="$IMAGE_DIR/e2e-rootfs.ext4"
+KERNEL="$IMAGE_DIR/vmlinux"
+FIRECRACKER="$BIN_DIR/firecracker"
+
+CONTROLPLANE_PID=""
+AGENT_PIDS=()
+KEEP_LAB="${FIREWORK_E2E_KEEP:-0}"
+SCENARIO_STATUS="failed"
+
+mkdir -p "$RUN_DIR" "$BIN_DIR" "$IMAGE_DIR" "$CONFIG_DIR" "$LOG_DIR" \
+  "$PKI_DIR" "$STATE_DIR" "$ROOTFS_MOUNT"
+chmod 755 "$RUN_DIR" "$LOG_DIR"
+
+log() {
+  printf '==> %s\n' "$*"
+}
+
+die() {
+  printf 'ERROR: %s\n' "$*" >&2
+  exit 1
+}
+
+case "$(uname -m)" in
+  x86_64) BIN_ARCH=amd64 ;;
+  aarch64|arm64) BIN_ARCH=arm64 ;;
+  *) die "unsupported Linux architecture in lab: $(uname -m)" ;;
+esac
+
+require_cmd() {
+  command -v "$1" >/dev/null 2>&1 || die "missing required command in lab: $1"
+}
+
+install_guest_tools() {
+  local missing=0
+  for command_name in aws curl git ip iptables jq openssl; do
+    if ! command -v "$command_name" >/dev/null 2>&1; then
+      missing=1
+    fi
+  done
+  if (( missing == 0 )); then
+    return
+  fi
+  command -v apt-get >/dev/null 2>&1 || die "lab tools are missing and apt-get is unavailable"
+  log "installing missing Linux lab tools"
+  export DEBIAN_FRONTEND=noninteractive
+  apt-get update -qq
+  apt-get install -y -qq awscli curl e2fsprogs git iproute2 iptables jq openssl
+}
+
+write_file() {
+  local path="$1"
+  mkdir -p "$(dirname "$path")"
+  cat > "$path"
+}
+
+cleanup_rootfs_mount() {
+  if mountpoint -q "$ROOTFS_MOUNT"; then
+    sync || true
+    umount "$ROOTFS_MOUNT" || true
+  fi
+}
+
+prepare_rootfs() {
+  local source_rootfs="${FIREWORK_E2E_ROOTFS:-}"
+  local source_kernel="${FIREWORK_E2E_KERNEL:-}"
+  local source_firecracker="${FIREWORK_E2E_FIRECRACKER_BIN:-}"
+  [[ -r "$source_rootfs" ]] || die "rootfs is not readable in the lab: $source_rootfs"
+  [[ -r "$source_kernel" ]] || die "kernel is not readable in the lab: $source_kernel"
+  [[ -x "$source_firecracker" ]] || die "Firecracker is not executable in the lab: $source_firecracker"
+
+  cp "$source_rootfs" "$GUEST_ROOTFS"
+  cp "$source_kernel" "$KERNEL"
+  cp "$source_firecracker" "$FIRECRACKER"
+  chmod 755 "$FIRECRACKER"
+
+  mount -o loop "$GUEST_ROOTFS" "$ROOTFS_MOUNT"
+  local busybox=""
+  for candidate in /bin/busybox /sbin/busybox /usr/bin/busybox; do
+    if [[ -x "$ROOTFS_MOUNT$candidate" ]]; then
+      busybox="$candidate"
+      break
+    fi
+  done
+  [[ -n "$busybox" ]] || die "rootfs must contain a static BusyBox binary"
+  [[ -x "$ROOTFS_MOUNT/bin/sh" || -x "$ROOTFS_MOUNT/bin/busybox" ]] || die "rootfs must provide /bin/sh"
+
+  install -D -m 0755 "${BIN_DIR}/fc-init-linux-$BIN_ARCH" "$ROOTFS_MOUNT/sbin/fc-init"
+  rm -f "$ROOTFS_MOUNT/sbin/init"
+  write_file "$ROOTFS_MOUNT/sbin/init" <<'EOF'
+#!/bin/sh
+set -eu
+busybox=/bin/busybox
+if [ ! -x "$busybox" ]; then busybox=/sbin/busybox; fi
+if [ ! -x "$busybox" ]; then busybox=/usr/bin/busybox; fi
+role="${FIREWORK_ROLE:-responder}"
+case "$role" in
+  responder) port=8080 ;;
+  caller)
+    target="${RESPONDER_URL:-}"
+    [ -n "$target" ] || exit 20
+    ready=0
+    attempt=0
+    while [ "$attempt" -lt 30 ]; do
+      if "$busybox" wget -q -O - "$target/health" >/dev/null 2>&1; then
+        ready=1
+        break
+      fi
+      attempt=$((attempt + 1))
+      sleep 1
+    done
+    [ "$ready" -eq 1 ] || exit 21
+    echo cross-node-ok > /tmp/cross-node-result
+    port=8081
+    ;;
+  *) exit 22 ;;
+esac
+exec "$busybox" httpd -f -p "$port" -h /www
+EOF
+  chmod 755 "$ROOTFS_MOUNT/sbin/init"
+  mkdir -p "$ROOTFS_MOUNT/www"
+  printf 'ok\n' > "$ROOTFS_MOUNT/www/health"
+  cleanup_rootfs_mount
+}
+
+setup_git_repo() {
+  mkdir -p "$CONFIG_DIR/repo/services"
+  write_file "$CONFIG_DIR/repo/defaults.yaml" <<EOF
+kernel: "$KERNEL"
+vcpus: 1
+memory_mb: 128
+kernel_args: "console=ttyS0 reboot=k panic=1 pci=off init=/sbin/fc-init"
+EOF
+  write_file "$CONFIG_DIR/repo/services/caller.yaml" <<EOF
+name: "caller"
+image: "$GUEST_ROOTFS"
+kernel: "$KERNEL"
+node_type: "general"
+network: true
+port_forwards:
+  - host_port: 18081
+    vm_port: 8081
+health_check:
+  type: "http"
+  port: 8081
+  path: "/health"
+  interval: "2s"
+  timeout: "1s"
+  retries: 5
+env:
+  FIREWORK_ROLE: "caller"
+cross_node_links:
+  - service: "responder"
+    env: "RESPONDER_URL"
+    host_port: 18080
+    protocol: "http"
+anti_affinity_group: "local-e2e"
+EOF
+  write_file "$CONFIG_DIR/repo/services/responder.yaml" <<EOF
+name: "responder"
+image: "$GUEST_ROOTFS"
+kernel: "$KERNEL"
+node_type: "general"
+network: true
+port_forwards:
+  - host_port: 18080
+    vm_port: 8080
+health_check:
+  type: "http"
+  port: 8080
+  path: "/health"
+  interval: "2s"
+  timeout: "1s"
+  retries: 5
+env:
+  FIREWORK_ROLE: "responder"
+anti_affinity_group: "local-e2e"
+EOF
+  git -C "$CONFIG_DIR/repo" init -b main >/dev/null 2>&1 || {
+    git -C "$CONFIG_DIR/repo" init >/dev/null
+    git -C "$CONFIG_DIR/repo" checkout -b main >/dev/null
+  }
+  git -C "$CONFIG_DIR/repo" config user.name firework-local-e2e
+  git -C "$CONFIG_DIR/repo" config user.email firework-local-e2e@localhost
+  git -C "$CONFIG_DIR/repo" add .
+  git -C "$CONFIG_DIR/repo" commit -m "local e2e workload" >/dev/null
+}
+
+setup_network() {
+  local namespace ip_address uplink
+  ip link add name fw-e2e-br type bridge
+  ip addr add 10.254.0.1/24 dev fw-e2e-br
+  ip link set fw-e2e-br up
+  sysctl -w net.ipv4.ip_forward=1 >/dev/null
+  for node in a b; do
+    namespace="fw-e2e-node-${node}"
+    if [[ "$node" == a ]]; then ip_address=10.254.0.11; else ip_address=10.254.0.12; fi
+    uplink="fw-e2e-${node}-up"
+    ip netns add "$namespace"
+    ip link add "fw-e2e-${node}" type veth peer name "$uplink"
+    ip link set "fw-e2e-${node}" master fw-e2e-br
+    ip link set "fw-e2e-${node}" up
+    ip link set "$uplink" netns "$namespace"
+    ip -n "$namespace" link set lo up
+    ip -n "$namespace" link set "$uplink" up
+    ip -n "$namespace" addr add "$ip_address/24" dev "$uplink"
+    ip -n "$namespace" route add default via 10.254.0.1
+    ip netns exec "$namespace" sysctl -w net.ipv4.ip_forward=1 >/dev/null
+    ip netns exec "$namespace" sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null
+    ip netns exec "$namespace" sysctl -w net.ipv4.conf.default.rp_filter=0 >/dev/null
+  done
+  ip route add 172.16.1.0/24 via 10.254.0.11 dev fw-e2e-br
+  ip route add 172.16.2.0/24 via 10.254.0.12 dev fw-e2e-br
+}
+
+write_controlplane_config() {
+  local operator_token="$1"
+  write_file "$CONFIG_DIR/controlplane.yaml" <<EOF
+role: "all"
+registry_listen_addr: "10.254.0.1:9443"
+events_listen_addr: "10.254.0.1:9444"
+api_listen_addr: "127.0.0.1:9445"
+operator_token: "$operator_token"
+ingress_domain: "local.e2e"
+state:
+  backend: "s3"
+  prefix: "cp/v1"
+  s3:
+    bucket: "$E2E_BUCKET"
+    region: "$AWS_REGION_VALUE"
+    force_path_style: false
+leader_lease_ttl: "6s"
+leader_renew_interval: "2s"
+node_stale_ttl: "15s"
+controller_tick: "2s"
+target_branch: "main"
+reconcile_on_start: true
+git_repo_url: "file://$CONFIG_DIR/repo"
+github_webhook_secret: "local-e2e-webhook-secret"
+tls:
+  cert_file: "$PKI_DIR/controlplane.crt"
+  key_file: "$PKI_DIR/controlplane.key"
+  client_ca_file: "$PKI_DIR/ca.crt"
+enrollment:
+  ca_file: "$PKI_DIR/ca.crt"
+  ca_key_file: "$PKI_DIR/ca.key"
+  node_cert_ttl: "2h"
+  bootstrap_tokens:
+    - token: "local-e2e-node-a-$FIREWORK_COMMIT"
+      node_id: "node-a"
+    - token: "local-e2e-node-b-$FIREWORK_COMMIT"
+      node_id: "node-b"
+EOF
+}
+
+write_agent_config() {
+  local node="$1" namespace_ip="$2" vm_subnet="$3" vm_gateway="$4" token="$5"
+  local api_port uplink
+  if [[ "$node" == node-a ]]; then
+    api_port=18081
+    uplink=fw-e2e-a-up
+  else
+    api_port=18082
+    uplink=fw-e2e-b-up
+  fi
+  write_file "$CONFIG_DIR/agent-${node}.yaml" <<EOF
+node_name: "$node"
+node_id: "$node"
+store_type: "s3"
+s3_bucket: "$E2E_BUCKET"
+s3_prefix: "cp/v1/"
+s3_region: "$AWS_REGION_VALUE"
+poll_interval: "2s"
+firecracker_bin: "$FIRECRACKER"
+state_dir: "$STATE_DIR/$node"
+images_dir: "$IMAGE_DIR"
+log_level: "debug"
+api_listen_addr: "$namespace_ip:$api_port"
+enable_health_checks: true
+enable_network_setup: true
+enable_capacity_check: false
+vm_subnet: "$vm_subnet"
+vm_gateway: "$vm_gateway"
+vm_bridge: "br-fw-$node"
+out_interface: "$uplink"
+registry_url: "https://10.254.0.1:9443"
+registry_server_name: "controlplane.local"
+registry_cert_file: "$STATE_DIR/$node/node.crt"
+registry_key_file: "$STATE_DIR/$node/node.key"
+registry_ca_file: "$PKI_DIR/ca.crt"
+registry_bootstrap_token: "$token"
+registry_heartbeat_interval: "2s"
+EOF
+}
+
+wait_http() {
+  local url="$1" ca_file="$2" auth_header="${3:-}" deadline=$((SECONDS + ${FIREWORK_E2E_TIMEOUT:-600}))
+  while (( SECONDS < deadline )); do
+    if [[ -n "$auth_header" ]]; then
+      if curl --silent --show-error --fail --cacert "$ca_file" -H "$auth_header" "$url" >/dev/null 2>&1; then return 0; fi
+    elif [[ -n "$ca_file" ]]; then
+      if curl --silent --show-error --fail --cacert "$ca_file" "$url" >/dev/null 2>&1; then return 0; fi
+    elif curl --silent --show-error --fail "$url" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 1
+  done
+  return 1
+}
+
+stop_processes() {
+  local pid
+  for pid in "${AGENT_PIDS[@]}" "$CONTROLPLANE_PID"; do
+    [[ -n "$pid" ]] || continue
+    kill -TERM "$pid" >/dev/null 2>&1 || true
+  done
+  for pid in "${AGENT_PIDS[@]}" "$CONTROLPLANE_PID"; do
+    [[ -n "$pid" ]] || continue
+    wait "$pid" >/dev/null 2>&1 || true
+  done
+  AGENT_PIDS=()
+  CONTROLPLANE_PID=""
+}
+
+delete_network() {
+  ip route del 172.16.1.0/24 via 10.254.0.11 dev fw-e2e-br >/dev/null 2>&1 || true
+  ip route del 172.16.2.0/24 via 10.254.0.12 dev fw-e2e-br >/dev/null 2>&1 || true
+  ip netns del fw-e2e-node-a >/dev/null 2>&1 || true
+  ip netns del fw-e2e-node-b >/dev/null 2>&1 || true
+  ip link del fw-e2e-br >/dev/null 2>&1 || true
+}
+
+write_manifest() {
+  local status="$1"
+  local agent_pids_json
+  agent_pids_json="$(printf '%s\n' "${AGENT_PIDS[@]}" | jq -Rsc 'split("\n") | map(select(length > 0) | tonumber)')"
+  jq -n \
+    --arg status "$status" \
+    --arg mode "$RUN_MODE" \
+    --arg commit "$FIREWORK_COMMIT" \
+    --arg bucket "$E2E_BUCKET" \
+    --arg region "$AWS_REGION_VALUE" \
+    --arg run_dir "$RUN_DIR" \
+    --arg controlplane_pid "${CONTROLPLANE_PID:-}" \
+    --argjson agent_pids "$agent_pids_json" \
+    '{status:$status,mode:$mode,firework_commit:$commit,bucket:$bucket,region:$region,run_dir:$run_dir,controlplane_pid:$controlplane_pid,agent_pids:$agent_pids}' \
+    > "$RUN_DIR/manifest.json"
+}
+
+cleanup() {
+  local status=$?
+  set +e
+  export CONTROLPLANE_CURL_URL="https://127.0.0.1:9445"
+  export CONTROLPLANE_CA_FILE="$PKI_DIR/ca.crt"
+  export CONTROLPLANE_OPERATOR_TOKEN="${OPERATOR_TOKEN:-}"
+  export E2E_BUCKET
+  export AGENT_ENDPOINTS="node-a=http://10.254.0.11:18081 node-b=http://10.254.0.12:18082"
+  "$SCRIPT_DIR/collect-diagnostics.sh" "$RUN_DIR" || true
+  if [[ "$status" -eq 0 ]]; then SCENARIO_STATUS="passed"; fi
+  cleanup_rootfs_mount
+  write_manifest "$SCENARIO_STATUS"
+  if [[ "$KEEP_LAB" != "1" ]]; then
+    stop_processes
+    delete_network
+  else
+    log "retaining local E2E lab at $RUN_DIR"
+    log "cleanup with: make validate-e2e-local-clean FIREWORK_E2E_MANIFEST=$RUN_DIR/manifest.json"
+  fi
+  exit "$status"
+}
+trap cleanup EXIT
+trap 'exit 130' INT TERM
+
+install_guest_tools
+for command_name in mountpoint mount umount install; do require_cmd "$command_name"; done
+[[ "$(id -u)" -eq 0 ]] || die "the Linux lab runner must execute as root"
+[[ -r /dev/kvm && -w /dev/kvm ]] || die "/dev/kvm is not readable and writable in the lab"
+[[ -e /dev/net/tun ]] || die "/dev/net/tun is required in the lab"
+
+log "preparing Firecracker assets"
+prepare_rootfs
+"$SCRIPT_DIR/generate-pki.sh" "$PKI_DIR"
+setup_git_repo
+setup_network
+
+OPERATOR_TOKEN="local-e2e-operator-$FIREWORK_COMMIT"
+write_controlplane_config "$OPERATOR_TOKEN"
+write_agent_config node-a 10.254.0.11 172.16.1.0/24 172.16.1.1 "local-e2e-node-a-$FIREWORK_COMMIT"
+write_agent_config node-b 10.254.0.12 172.16.2.0/24 172.16.2.1 "local-e2e-node-b-$FIREWORK_COMMIT"
+
+log "starting combined control plane"
+"$BIN_DIR/firework-controlplane-linux-$BIN_ARCH" --config "$CONFIG_DIR/controlplane.yaml" > "$LOG_DIR/controlplane.log" 2>&1 &
+CONTROLPLANE_PID=$!
+wait_http "https://127.0.0.1:9445/healthz" "$PKI_DIR/ca.crt" || die "control-plane API did not become healthy"
+
+log "starting two isolated agents"
+ip netns exec fw-e2e-node-a "$BIN_DIR/firework-agent-linux-$BIN_ARCH" --config "$CONFIG_DIR/agent-node-a.yaml" > "$LOG_DIR/node-a.log" 2>&1 &
+AGENT_PIDS+=("$!")
+ip netns exec fw-e2e-node-b "$BIN_DIR/firework-agent-linux-$BIN_ARCH" --config "$CONFIG_DIR/agent-node-b.yaml" > "$LOG_DIR/node-b.log" 2>&1 &
+AGENT_PIDS+=("$!")
+wait_http "http://10.254.0.11:18081/healthz" "" || die "node-a API did not become healthy"
+wait_http "http://10.254.0.12:18082/healthz" "" || die "node-b API did not become healthy"
+
+nodes_url="https://127.0.0.1:9445/v1/nodes"
+services_url="https://127.0.0.1:9445/v1/services"
+auth_header="Authorization: Bearer $OPERATOR_TOKEN"
+deadline=$((SECONDS + ${FIREWORK_E2E_TIMEOUT:-600}))
+nodes_json=""
+services_json=""
+while (( SECONDS < deadline )); do
+  nodes_json="$(curl --silent --show-error --fail --cacert "$PKI_DIR/ca.crt" -H "$auth_header" "$nodes_url" 2>/dev/null || true)"
+  services_json="$(curl --silent --show-error --fail --cacert "$PKI_DIR/ca.crt" -H "$auth_header" "$services_url" 2>/dev/null || true)"
+  if jq -e '.count == 2 and ([.items[].node_id] | unique | length == 2) and all(.items[]; .state == "ready")' <<<"$nodes_json" >/dev/null 2>&1 \
+    && jq -e '.count == 2 and all(.items[]; .state == "running" and .health == "healthy") and ([.items[].node] | unique | length == 2)' <<<"$services_json" >/dev/null 2>&1; then
+    break
+  fi
+  sleep 2
+done
+
+jq -e '.count == 2 and ([.items[].node_id] | unique | length == 2)' <<<"$nodes_json" >/dev/null \
+  || die "two nodes did not register: $nodes_json"
+jq -e '.count == 2 and all(.items[]; .state == "running" and .health == "healthy") and ([.items[].node] | unique | length == 2)' <<<"$services_json" >/dev/null \
+  || die "two healthy cross-node services did not converge: $services_json"
+
+caller_node="$(jq -r '.items[] | select(.name == "caller") | .node' <<<"$services_json")"
+case "$caller_node" in
+  node-a) caller_endpoint="http://10.254.0.11:18081" ;;
+  node-b) caller_endpoint="http://10.254.0.12:18082" ;;
+  *) die "caller service has no valid node placement: $caller_node" ;;
+esac
+curl --silent --show-error --fail "$caller_endpoint/health" >/dev/null \
+  || die "caller endpoint did not become reachable; cross-node link likely failed"
+
+for node in node-a node-b; do
+  aws s3api head-object --bucket "$E2E_BUCKET" --key "cp/v1/nodes/$node.yaml" >/dev/null \
+    || die "real S3 bucket has no rendered config for $node"
+done
+aws s3api list-objects-v2 --bucket "$E2E_BUCKET" --prefix cp/v1/ --output json > "$RUN_DIR/s3-inventory.json"
+printf '%s\n' "$nodes_json" > "$RUN_DIR/nodes.json"
+printf '%s\n' "$services_json" > "$RUN_DIR/services.json"
+SCENARIO_STATUS="passed"
+log "local two-node E2E passed"
+log "node placement: $(jq -r '.items[] | [.name, .node] | @tsv' <<<"$services_json" | tr '\n' ' ')"
diff --git a/validation/e2e-local/scripts/validate.sh b/validation/e2e-local/scripts/validate.sh
new file mode 100755
index 0000000..01960bb
--- /dev/null
+++ b/validation/e2e-local/scripts/validate.sh
@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+GUEST_RUNNER="$SCRIPT_DIR/run-guest.sh"
+
+log() {
+  printf '==> %s\n' "$*"
+}
+
+die() {
+  printf 'ERROR: %s\n' "$*" >&2
+  exit 1
+}
+
+require_cmd() {
+  command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
+}
+
+require_cmd aws
+require_cmd curl
+require_cmd git
+require_cmd go
+require_cmd jq
+require_cmd make
+
+if [[ -z "${AWS_ACCESS_KEY_ID:-}" && -n "${AWS_PROFILE:-}" ]]; then
+  eval "$(aws configure export-credentials --profile "$AWS_PROFILE" --format env)"
+  unset AWS_PROFILE
+fi
+
+export AWS_REGION="${AWS_REGION:-${AWS_DEFAULT_REGION:-}}"
+export AWS_DEFAULT_REGION="$AWS_REGION"
+[[ -n "$AWS_REGION" ]] || die "AWS_REGION or AWS_DEFAULT_REGION is required"
+aws sts get-caller-identity >/dev/null || die "AWS credentials are not usable"
+
+account_id="$(aws sts get-caller-identity --query Account --output text)"
+run_id="$(date -u +%Y%m%dt%H%M%S)-$$"
+bucket="firework-local-e2e-${account_id}-${run_id//[^a-zA-Z0-9-]/-}"
+bucket="${bucket:0:63}"
+
+case "$(uname -s)" in
+  Darwin)
+    host_arch="arm64"
+    default_mode="lima"
+    ;;
+  Linux)
+    case "$(uname -m)" in
+      x86_64) host_arch="amd64" ;;
+      aarch64|arm64) host_arch="arm64" ;;
+      *) die "unsupported Linux architecture: $(uname -m)" ;;
+    esac
+    default_mode="linux"
+    ;;
+  *) die "unsupported host OS: $(uname -s)" ;;
+esac
+mode="${FIREWORK_E2E_MODE:-$default_mode}"
+[[ "$mode" == linux || "$mode" == lima ]] || die "FIREWORK_E2E_MODE must be linux or lima"
+if [[ "$mode" == linux && "$(uname -s)" != Linux ]]; then
+  die "FIREWORK_E2E_MODE=linux requires a Linux host"
+fi
+if [[ "$mode" == lima && "$(uname -s)" != Darwin ]]; then
+  die "FIREWORK_E2E_MODE=lima requires macOS"
+fi
+
+[[ -n "${FIREWORK_E2E_FIRECRACKER_BIN:-}" && -x "$FIREWORK_E2E_FIRECRACKER_BIN" ]] || die "set executable FIREWORK_E2E_FIRECRACKER_BIN"
+[[ -n "${FIREWORK_E2E_KERNEL:-}" && -r "$FIREWORK_E2E_KERNEL" ]] || die "set readable FIREWORK_E2E_KERNEL"
+[[ -n "${FIREWORK_E2E_ROOTFS:-}" && -r "$FIREWORK_E2E_ROOTFS" ]] || die "set readable FIREWORK_E2E_ROOTFS"
+
+log "building Linux Firework binaries"
+(cd "$REPO_ROOT" && make "build-linux-$host_arch" >/dev/null)
+
+commit="$(git -C "$REPO_ROOT" rev-parse HEAD)"
+if [[ -n "${FIREWORK_E2E_WORKDIR:-}" ]]; then
+  host_run_dir="$FIREWORK_E2E_WORKDIR"
+else
+  host_run_dir="$(mktemp -d "${TMPDIR:-/tmp}/firework-e2e-local.XXXXXX")"
+fi
+mkdir -p "$host_run_dir"
+
+instance=""
+guest_run_dir="/tmp/firework-e2e-local-$run_id"
+status=1
+bucket_created=0
+
+cleanup_bucket() {
+  local cleanup_status=0
+  if [[ "$bucket_created" -ne 1 || "${FIREWORK_E2E_KEEP:-0}" == 1 ]]; then
+    return 0
+  fi
+  log "deleting real S3 bucket $bucket"
+  aws s3 rm "s3://$bucket" --recursive >/dev/null 2>&1 || cleanup_status=1
+  aws s3api delete-bucket --bucket "$bucket" >/dev/null 2>&1 || cleanup_status=1
+  if [[ "$cleanup_status" -ne 0 ]]; then
+    printf 'WARNING: bucket cleanup failed; inspect and delete only %s manually\n' "$bucket" >&2
+  fi
+  return "$cleanup_status"
+}
+
+cleanup_runtime() {
+  set +e
+  if [[ "$mode" == lima && -n "$instance" ]]; then
+    if [[ "${FIREWORK_E2E_KEEP:-0}" == 1 ]]; then
+      limactl shell --yes "$instance" -- sudo -n chmod 644 "$guest_run_dir/manifest.json" >/dev/null 2>&1 || true
+      limactl copy "$instance:$guest_run_dir/manifest.json" "$host_run_dir/manifest.json" >/dev/null 2>&1 || true
+    else
+      limactl shell --yes "$instance" -- sudo -n chmod -R a+rX "$guest_run_dir" >/dev/null 2>&1 || true
+      limactl copy --recursive "$instance:$guest_run_dir" "$host_run_dir/guest" >/dev/null 2>&1 || true
+    fi
+    if [[ "${FIREWORK_E2E_KEEP:-0}" == 1 && -f "$host_run_dir/manifest.json" ]]; then
+      jq --arg instance "$instance" '. + {lima_instance:$instance}' \
+        "$host_run_dir/manifest.json" > "$host_run_dir/manifest.tmp"
+      mv "$host_run_dir/manifest.tmp" "$host_run_dir/manifest.json"
+    fi
+    if [[ "${FIREWORK_E2E_KEEP:-0}" != 1 ]]; then
+      limactl stop --force "$instance" >/dev/null 2>&1 || true
+      limactl delete --force "$instance" >/dev/null 2>&1 || true
+    else
+      log "retaining Lima instance $instance"
+    fi
+  fi
+  cleanup_bucket || status=1
+  if [[ "${FIREWORK_E2E_KEEP:-0}" == 1 ]]; then
+    log "retained host artifacts: $host_run_dir"
+    log "retained bucket: $bucket"
+  else
+    log "local E2E artifacts: $host_run_dir"
+  fi
+  exit "$status"
+}
+trap cleanup_runtime EXIT INT TERM
+
+log "creating real S3 bucket $bucket"
+if [[ "$AWS_REGION" == us-east-1 ]]; then
+  aws s3api create-bucket --bucket "$bucket" --region "$AWS_REGION" >/dev/null
+else
+  aws s3api create-bucket --bucket "$bucket" --region "$AWS_REGION" \
+    --create-bucket-configuration "LocationConstraint=$AWS_REGION" >/dev/null
+fi
+bucket_created=1
+
+if [[ "$mode" == linux ]]; then
+  "$SCRIPT_DIR/check-env.sh"
+  sudo_env="AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_REGION,AWS_DEFAULT_REGION,AWS_EC2_METADATA_DISABLED,FIREWORK_E2E_KEEP,FIREWORK_E2E_TIMEOUT,FIREWORK_E2E_FIRECRACKER_BIN,FIREWORK_E2E_KERNEL,FIREWORK_E2E_ROOTFS"
+  if [[ "$(id -u)" -eq 0 ]]; then
+    bash "$GUEST_RUNNER" "$host_run_dir" "$bucket" "$AWS_REGION" "$commit" linux
+  else
+    require_cmd sudo
+    sudo --preserve-env="$sudo_env" bash "$GUEST_RUNNER" "$host_run_dir" "$bucket" "$AWS_REGION" "$commit" linux
+  fi
+  status=$?
+else
+  require_cmd limactl
+  [[ "$(uname -m)" == arm64 ]] || die "Lima mode currently requires an Apple Silicon host"
+
+  instance="firework-e2e-$run_id"
+  log "starting Lima instance $instance"
+  limactl start --yes --name "$instance" --plain --vm-type=vz --nested-virt \
+    --cpus 4 --memory 8 --disk 30 >/dev/null
+  limactl shell --yes "$instance" -- sudo -n apt-get update -qq >/dev/null
+  limactl shell --yes "$instance" -- sudo -n apt-get install -y -qq awscli curl e2fsprogs git iproute2 iptables jq openssl >/dev/null
+
+  guest_root="$guest_run_dir"
+  limactl shell --yes "$instance" -- mkdir -p "$guest_root/bin" "$guest_root/images" "$guest_root/scripts"
+  limactl copy "$GUEST_RUNNER" "$instance:$guest_root/run-guest.sh"
+  limactl copy "$SCRIPT_DIR/generate-pki.sh" "$instance:$guest_root/scripts/generate-pki.sh"
+  limactl copy "$SCRIPT_DIR/collect-diagnostics.sh" "$instance:$guest_root/scripts/collect-diagnostics.sh"
+  limactl copy "$REPO_ROOT/bin/firework-agent-linux-arm64" "$instance:$guest_root/bin/firework-agent-linux-arm64"
+  limactl copy "$REPO_ROOT/bin/firework-controlplane-linux-arm64" "$instance:$guest_root/bin/firework-controlplane-linux-arm64"
+  limactl copy "$REPO_ROOT/bin/fc-init-linux-arm64" "$instance:$guest_root/bin/fc-init-linux-arm64"
+  limactl copy "$FIREWORK_E2E_FIRECRACKER_BIN" "$instance:$guest_root/images/firecracker"
+  limactl copy "$FIREWORK_E2E_KERNEL" "$instance:$guest_root/images/vmlinux-source"
+  limactl copy "$FIREWORK_E2E_ROOTFS" "$instance:$guest_root/images/rootfs-source.ext4"
+  sudo_env="AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_REGION,AWS_DEFAULT_REGION,AWS_EC2_METADATA_DISABLED,FIREWORK_E2E_KEEP,FIREWORK_E2E_TIMEOUT"
+  LIMA_SHELLENV_ALLOW="+AWS_ACCESS_KEY_ID,+AWS_SECRET_ACCESS_KEY,+AWS_SESSION_TOKEN,+AWS_REGION,+AWS_DEFAULT_REGION" \
+    limactl shell --yes --preserve-env "$instance" -- \
+      sudo -n --preserve-env="$sudo_env" env \
+        FIREWORK_E2E_FIRECRACKER_BIN="$guest_root/images/firecracker" \
+        FIREWORK_E2E_KERNEL="$guest_root/images/vmlinux-source" \
+        FIREWORK_E2E_ROOTFS="$guest_root/images/rootfs-source.ext4" \
+        FIREWORK_E2E_SCRIPT_DIR="$guest_root/scripts" \
+        bash "$guest_root/run-guest.sh" "$guest_root" "$bucket" "$AWS_REGION" "$commit" lima
+  status=$?
+fi

From 410a59e82d4c235fe075d07af73dd3893e80ef86 Mon Sep 17 00:00:00 2001
From: Artem Nikitin <artemnikitin@artemnikitin.com>
Date: Thu, 6 Aug 2026 20:09:35 +0200
Subject: [PATCH 2/2] Implement milestones 0 and 1

---
 internal/vm/manager.go                        |   7 +
 internal/vm/manager_test.go                   |   3 +
 validation/e2e-local/README.md                | 112 ++--
 validation/e2e-local/scripts/check-env.sh     |   7 -
 .../e2e-local/scripts/prepare-assets.sh       |  66 +++
 validation/e2e-local/scripts/run-guest.sh     | 509 +++++++++++++-----
 validation/e2e-local/scripts/validate.sh      |  89 ++-
 7 files changed, 578 insertions(+), 215 deletions(-)
 create mode 100755 validation/e2e-local/scripts/prepare-assets.sh

diff --git a/internal/vm/manager.go b/internal/vm/manager.go
index a0a32c7..b51ec11 100644
--- a/internal/vm/manager.go
+++ b/internal/vm/manager.go
@@ -538,6 +538,10 @@ func (m *Manager) writeVMConfig(vmDir string, svc config.ServiceConfig, prepared
 		Drives:            drives,
 		MachineConfig:     firecrackerMachineConfig{VCPUCount: svc.VCPUs, MemSizeMiB: svc.MemoryMB},
 		NetworkInterfaces: networkInterfaces,
+		// The VirtIO-RNG device prevents guests without a usable hardware
+		// random source from blocking application startup on /dev/random.
+		// This matters for arm64 guests nested inside Lima/VZ in particular.
+		Entropy: &firecrackerEntropyDevice{},
 	}
 	configJSON, err := json.MarshalIndent(vmConfig, "", "  ")
 	if err != nil {
@@ -556,6 +560,7 @@ type firecrackerConfig struct {
 	Drives            []firecrackerDrive            `json:"drives"`
 	MachineConfig     firecrackerMachineConfig      `json:"machine-config"`
 	NetworkInterfaces []firecrackerNetworkInterface `json:"network-interfaces,omitempty"`
+	Entropy           *firecrackerEntropyDevice     `json:"entropy,omitempty"`
 }
 
 type firecrackerBootSource struct {
@@ -575,6 +580,8 @@ type firecrackerMachineConfig struct {
 	MemSizeMiB int `json:"mem_size_mib"`
 }
 
+type firecrackerEntropyDevice struct{}
+
 type firecrackerNetworkInterface struct {
 	IfaceID     string `json:"iface_id"`
 	GuestMAC    string `json:"guest_mac"`
diff --git a/internal/vm/manager_test.go b/internal/vm/manager_test.go
index b614b82..a6a6432 100644
--- a/internal/vm/manager_test.go
+++ b/internal/vm/manager_test.go
@@ -75,6 +75,9 @@ func TestWriteVMConfigAddsDeterministicVolumeDrivesAndPayload(t *testing.T) {
 	if len(cfg.Drives) != 3 || cfg.Drives[1].PathOnHost != "/a.ext4" || cfg.Drives[2].PathOnHost != "/z.ext4" {
 		t.Fatalf("unexpected drives: %#v", cfg.Drives)
 	}
+	if cfg.Entropy == nil {
+		t.Fatal("expected a VirtIO-RNG entropy device")
+	}
 	fields := strings.Fields(cfg.BootSource.BootArgs)
 	var encoded string
 	for i, field := range fields {
diff --git a/validation/e2e-local/README.md b/validation/e2e-local/README.md
index c538b1d..addb10f 100644
--- a/validation/e2e-local/README.md
+++ b/validation/e2e-local/README.md
@@ -1,67 +1,87 @@
-# Local two-node E2E validation
+# Local stateful two-node E2E validation
 
-This harness runs one complete local Firework setup:
+This harness runs one combined Firework setup inside an arm64 Lima Linux
+guest on Apple Silicon:
 
 ```text
-one control-plane process (registry + events + controller + API)
+one control plane (registry + events + controller + API)
                          |
-                 real AWS S3 bucket
+                 real per-run S3 bucket
                          |
-          node-a network namespace -- node-b network namespace
-             firework-agent             firework-agent
-                  |                            |
-             Firecracker VM               Firecracker VM
+       node-a namespace ---------------- node-b namespace
+       firework-agent                    firework-agent
+       Firecracker microVMs              Firecracker microVMs
 ```
 
-The control plane reconciles a disposable local Git repository. It publishes
-the desired state and rendered node configs into a unique real S3 bucket. Both
-agents enroll over mTLS, consume their rendered configs from that bucket, and
-run one Firecracker VM each. The caller VM is deliberately placed on the
-opposite node from the responder VM and reaches it through a rendered
-cross-node link.
+The workload uses the arm64 rootfs images produced by
+`firework-gitops-example`: Elasticsearch and Kibana start together on node-a,
+then the same desired state is changed to anti-affine placement and Kibana
+moves to node-b. Both phases require eventual `running`/`healthy` convergence;
+short startup or movement downtime is expected and allowed. Elasticsearch is
+also driven to cluster status `green` after its single-node replica setting is
+adjusted for this validation.
 
-This is local developer validation for now. CI and the unprivileged-build /
-privileged-lab handoff are the next milestone.
+The run additionally checks the real S3 state/rendered-config path, local
+volume creation and reuse, agent restart/adoption, explicit empty desired
+state, stale-node visibility and recovery, cross-node routing, and final
+per-node port ownership. CI, Linux-native execution, and hosted-KVM probing
+are Milestone 2 work and are not required by this local command today.
 
 ## Prerequisites
 
-- Linux with root, `/dev/kvm`, `/dev/net/tun`, `ip`, and `iptables`; or
-  Apple Silicon macOS with Lima, nested virtualization, and a Linux guest
-  that exposes `/dev/kvm`.
+- Apple Silicon macOS with Lima 2.x and a Lima `vz` guest that exposes
+  readable/writable `/dev/kvm` and `/dev/net/tun`.
 - Go and the repository build toolchain.
-- AWS credentials with permission to create, list, write, and delete a
-  disposable S3 bucket. The runner creates and deletes its own bucket on every
-  run; it refuses to reuse an existing bucket name.
-- A Linux Firecracker binary, an uncompressed Linux kernel, and an ext4
-  rootfs containing BusyBox. The runner copies `fc-init` and the disposable
-  E2E init script into a temporary copy of the rootfs, so the supplied rootfs
-  is not modified.
+- AWS credentials able to create/list/write/delete a disposable S3 bucket and
+  read the workload-image bucket.
+- Access to the existing arm64 GitOps image bucket. The default is
+  `artemnikitin-firework-images`; override it with
+  `FIREWORK_E2E_IMAGES_BUCKET` when needed.
 
-Set these paths before running:
+The harness downloads and verifies Firecracker 1.12.0 arm64 and uses the pinned
+Firecracker CI kernel `firecracker-ci/v1.12/aarch64/vmlinux-5.10.233`. The VM
+configuration enables Firecracker's VirtIO-RNG device so Java/Node workloads
+do not wait indefinitely for guest entropy. Override either asset pin only for
+an intentional compatibility investigation. The two workload rootfs images
+are not copied into the repository or manually modified: the agents download
+them through their production S3 image-sync path.
+
+## Run
 
 ```bash
-export FIREWORK_E2E_FIRECRACKER_BIN=/path/to/firecracker
-export FIREWORK_E2E_KERNEL=/path/to/vmlinux
-export FIREWORK_E2E_ROOTFS=/path/to/rootfs.ext4
 export AWS_REGION=us-east-1
+export FIREWORK_E2E_AWS_PROFILE=artemnikitin
 make validate-e2e-local
 ```
 
-When `AWS_PROFILE` is used instead of environment credentials, the host
-runner exports that profile's current credentials before entering Lima. This
-also supports short-lived credentials obtained through the AWS CLI.
-
-Useful local-only options:
+Useful local options:
 
-- `FIREWORK_E2E_MODE=linux` forces native Linux execution.
-- `FIREWORK_E2E_MODE=lima` forces the macOS Lima adapter.
-- `FIREWORK_E2E_KEEP=1` retains the lab and prints a manifest path and cleanup
-  command. Use `make validate-e2e-local-clean FIREWORK_E2E_MANIFEST=...` after
-  inspection.
-- `FIREWORK_E2E_TIMEOUT=600` changes the bounded scenario timeout.
+- `FIREWORK_E2E_KEEP=1` retains the Lima guest, logs, manifest and disposable
+  S3 bucket for inspection. Clean it with
+  `make validate-e2e-local-clean FIREWORK_E2E_MANIFEST=<manifest>`.
+- `FIREWORK_E2E_TIMEOUT=1800` changes the bounded scenario timeout. The default
+  is intentionally generous because these production-sized rootfs images can
+  take several minutes to initialize under nested virtualization.
+- `FIREWORK_E2E_LIMA_CPUS=8`, `FIREWORK_E2E_LIMA_MEMORY_GB=12`, and
+  `FIREWORK_E2E_LIMA_DISK_GB=60` size the local guest.
+- By default Elasticsearch gets 4 vCPUs and 6 GiB, while Kibana gets 2 vCPUs
+  and 4 GiB. Override these with `FIREWORK_E2E_ES_VCPUS`,
+  `FIREWORK_E2E_ES_MEMORY_MB`, `FIREWORK_E2E_KIBANA_VCPUS`, and
+  `FIREWORK_E2E_KIBANA_MEMORY_MB` when the host has different capacity.
+- `FIREWORK_E2E_HEALTH_RETRIES=80` controls the startup/restart threshold.
+  `FIREWORK_E2E_ES_JAVA_OPTS=-Xmx1g` is the compatibility default for the
+  currently published GitOps Elasticsearch image; a rebuilt image with the
+  current `fc-init` can use a normal multi-option value.
+- `FIREWORK_E2E_VOLUME_SIZE=2Gi` and
+  `FIREWORK_E2E_STORAGE_CAPACITY=8Gi` adjust the disposable local volume
+  pool.
+- `FIREWORK_E2E_ES_IMAGE_KEY` and `FIREWORK_E2E_KIBANA_IMAGE_KEY` select
+  alternate objects with the same GitOps rootfs contract.
+- `FIREWORK_E2E_FIRECRACKER_BIN` and `FIREWORK_E2E_KERNEL` optionally provide
+  local asset overrides; otherwise the pinned downloads are used.
 
-The runner writes logs, redacted configuration diagnostics, S3 object
-inventory, network state, and a run manifest under a temporary directory. AWS
-credentials are passed to the runtime through the process environment and are
-never written to the manifest or generated configuration files; the generated
-files do contain short-lived local mTLS/bootstrap tokens needed by the lab.
+The runner creates a unique real S3 bucket for control-plane state and
+rendered node configs, records image/asset provenance, collects diagnostics
+before teardown, and deletes the bucket and Lima guest unless retention is
+requested. AWS credentials are passed through the process environment and are
+not written to the manifest or generated configuration files.
diff --git a/validation/e2e-local/scripts/check-env.sh b/validation/e2e-local/scripts/check-env.sh
index 7be8255..51b149f 100755
--- a/validation/e2e-local/scripts/check-env.sh
+++ b/validation/e2e-local/scripts/check-env.sh
@@ -18,13 +18,6 @@ require_cmd openssl
 
 [[ -n "${AWS_REGION:-${AWS_DEFAULT_REGION:-}}" ]] || die "AWS_REGION or AWS_DEFAULT_REGION is required"
 
-[[ -n "${FIREWORK_E2E_FIRECRACKER_BIN:-}" ]] || die "FIREWORK_E2E_FIRECRACKER_BIN is required"
-[[ -x "${FIREWORK_E2E_FIRECRACKER_BIN}" ]] || die "Firecracker binary is not executable: ${FIREWORK_E2E_FIRECRACKER_BIN}"
-[[ -n "${FIREWORK_E2E_KERNEL:-}" ]] || die "FIREWORK_E2E_KERNEL is required"
-[[ -r "${FIREWORK_E2E_KERNEL}" ]] || die "kernel is not readable: ${FIREWORK_E2E_KERNEL}"
-[[ -n "${FIREWORK_E2E_ROOTFS:-}" ]] || die "FIREWORK_E2E_ROOTFS is required"
-[[ -r "${FIREWORK_E2E_ROOTFS}" ]] || die "rootfs is not readable: ${FIREWORK_E2E_ROOTFS}"
-
 case "$(uname -s)" in
   Darwin)
     require_cmd limactl
diff --git a/validation/e2e-local/scripts/prepare-assets.sh b/validation/e2e-local/scripts/prepare-assets.sh
new file mode 100755
index 0000000..ea26058
--- /dev/null
+++ b/validation/e2e-local/scripts/prepare-assets.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+RUN_DIR=${1:?usage: prepare-assets.sh RUN_DIR}
+IMAGE_DIR="$RUN_DIR/images"
+BIN_DIR="$RUN_DIR/bin"
+FIRECRACKER_VERSION="${FIREWORK_E2E_FIRECRACKER_VERSION:-1.12.0}"
+FIRECRACKER_ARCH="${FIREWORK_E2E_FIRECRACKER_ARCH:-aarch64}"
+FIRECRACKER_TARBALL="firecracker-v${FIRECRACKER_VERSION}-${FIRECRACKER_ARCH}.tgz"
+FIRECRACKER_SHA256="${FIREWORK_E2E_FIRECRACKER_SHA256:-55f3e76c6a16128e91aea1d2ed3d436f5d4e2e9547bfdd226ce570a89cd48921}"
+KERNEL_KEY="${FIREWORK_E2E_KERNEL_KEY:-firecracker-ci/v1.12/aarch64/vmlinux-5.10.233}"
+
+log() {
+  printf '==> %s\n' "$*"
+}
+
+die() {
+  printf 'ERROR: %s\n' "$*" >&2
+  exit 1
+}
+
+mkdir -p "$IMAGE_DIR" "$BIN_DIR"
+
+if [[ ! -x "$BIN_DIR/firecracker" ]]; then
+  tmp_dir="$(mktemp -d "$RUN_DIR/firecracker-download.XXXXXX")"
+  trap 'rm -rf "$tmp_dir"' EXIT
+  archive="$tmp_dir/$FIRECRACKER_TARBALL"
+  url="https://github.com/firecracker-microvm/firecracker/releases/download/v${FIRECRACKER_VERSION}/${FIRECRACKER_TARBALL}"
+  log "downloading pinned Firecracker $FIRECRACKER_VERSION ($FIRECRACKER_ARCH)"
+  curl --fail --silent --show-error --location "$url" --output "$archive"
+  printf '%s  %s\n' "$FIRECRACKER_SHA256" "$archive" | sha256sum --check --status - \
+    || die "Firecracker archive checksum mismatch: $url"
+  tar --extract --gzip --file "$archive" --directory "$tmp_dir"
+  extracted="$tmp_dir/release-v${FIRECRACKER_VERSION}-${FIRECRACKER_ARCH}/firecracker-v${FIRECRACKER_VERSION}-${FIRECRACKER_ARCH}"
+  [[ -x "$extracted" ]] || die "Firecracker archive did not contain $extracted"
+  install -m 0755 "$extracted" "$BIN_DIR/firecracker"
+fi
+
+[[ -x "$BIN_DIR/firecracker" ]] || die "Firecracker is not executable: $BIN_DIR/firecracker"
+
+kernel="$IMAGE_DIR/vmlinux"
+if [[ ! -r "$kernel" ]]; then
+  log "downloading pinned Firecracker kernel $KERNEL_KEY"
+  curl --fail --silent --show-error --location \
+    "https://s3.amazonaws.com/spec.ccfc.min/$KERNEL_KEY" --output "$kernel"
+  chmod 0644 "$kernel"
+fi
+
+[[ -s "$kernel" ]] || die "kernel is empty: $kernel"
+
+firecracker_version="$("$BIN_DIR/firecracker" --version 2>&1 | head -n 1)"
+kernel_sha256="$(sha256sum "$kernel" | awk '{print $1}')"
+firecracker_sha256="$(sha256sum "$BIN_DIR/firecracker" | awk '{print $1}')"
+cat > "$RUN_DIR/assets-manifest.json" <<EOF
+{
+  "firecracker_version": "$FIRECRACKER_VERSION",
+  "firecracker_arch": "$FIRECRACKER_ARCH",
+  "firecracker_release_sha256": "$FIRECRACKER_SHA256",
+  "firecracker_binary_sha256": "$firecracker_sha256",
+  "firecracker_reported_version": "$firecracker_version",
+  "kernel_key": "$KERNEL_KEY",
+  "kernel_sha256": "$kernel_sha256"
+}
+EOF
+
+log "prepared Firecracker and kernel assets"
diff --git a/validation/e2e-local/scripts/run-guest.sh b/validation/e2e-local/scripts/run-guest.sh
index 5c88e84..af1296b 100755
--- a/validation/e2e-local/scripts/run-guest.sh
+++ b/validation/e2e-local/scripts/run-guest.sh
@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
-RUN_DIR=${1:?usage: run-guest.sh RUN_DIR BUCKET REGION COMMIT MODE}
+RUN_DIR=${1:?usage: run-guest.sh RUN_DIR BUCKET REGION COMMIT MODE IMAGES_BUCKET}
 E2E_BUCKET=${2:?missing bucket}
 AWS_REGION_VALUE=${3:?missing AWS region}
 FIREWORK_COMMIT=${4:?missing Firework commit}
 RUN_MODE=${5:?missing run mode}
+IMAGES_BUCKET=${6:?missing workload image bucket}
 
 export AWS_REGION="$AWS_REGION_VALUE"
 export AWS_DEFAULT_REGION="$AWS_REGION_VALUE"
@@ -19,9 +20,30 @@ LOG_DIR="$RUN_DIR/logs"
 PKI_DIR="$RUN_DIR/pki"
 STATE_DIR="$RUN_DIR/state"
 ROOTFS_MOUNT="$RUN_DIR/rootfs-mount"
-GUEST_ROOTFS="$IMAGE_DIR/e2e-rootfs.ext4"
+DIRECT_BIN_DIR="$RUN_DIR/direct-bin"
 KERNEL="$IMAGE_DIR/vmlinux"
 FIRECRACKER="$BIN_DIR/firecracker"
+ASSET_SCRIPT="${FIREWORK_E2E_ASSET_SCRIPT:-$SCRIPT_DIR/prepare-assets.sh}"
+ES_IMAGE_KEY="${FIREWORK_E2E_ES_IMAGE_KEY:-tenant-2-elasticsearch-rootfs.ext4}"
+KIBANA_IMAGE_KEY="${FIREWORK_E2E_KIBANA_IMAGE_KEY:-tenant-2-kibana-rootfs.ext4}"
+VOLUME_SIZE="${FIREWORK_E2E_VOLUME_SIZE:-2Gi}"
+STORAGE_CAPACITY="${FIREWORK_E2E_STORAGE_CAPACITY:-8Gi}"
+HEALTH_RETRIES="${FIREWORK_E2E_HEALTH_RETRIES:-80}"
+E2E_TIMEOUT="${FIREWORK_E2E_TIMEOUT:-1800}"
+ES_VCPUS="${FIREWORK_E2E_ES_VCPUS:-${FIREWORK_E2E_SERVICE_VCPUS:-4}}"
+ES_MEMORY_MB="${FIREWORK_E2E_ES_MEMORY_MB:-${FIREWORK_E2E_SERVICE_MEMORY_MB:-6144}}"
+KIBANA_VCPUS="${FIREWORK_E2E_KIBANA_VCPUS:-${FIREWORK_E2E_SERVICE_VCPUS:-2}}"
+KIBANA_MEMORY_MB="${FIREWORK_E2E_KIBANA_MEMORY_MB:-${FIREWORK_E2E_SERVICE_MEMORY_MB:-4096}}"
+# Keep the boot argument explicit for guests that expose a usable arm64 CPU
+# random source. The Firecracker VM config also enables VirtIO-RNG for VZ/KVM
+# environments where that source is not available.
+KERNEL_BOOT_ARGS="console=ttyS0 reboot=k panic=1 pci=off random.trust_cpu=on"
+# Existing GitOps images bundle an fc-init that predates firework.env64, so
+# keep the default override to one whitespace-free JVM option. A rebuilt image
+# with the current fc-init can override this with multiple options.
+ES_JAVA_OPTS="${FIREWORK_E2E_ES_JAVA_OPTS--Xmx1g}"
+WEBHOOK_SECRET="local-e2e-webhook-secret"
+ROOT_OUT_INTERFACE=""
 
 CONTROLPLANE_PID=""
 AGENT_PIDS=()
@@ -53,7 +75,7 @@ require_cmd() {
 
 install_guest_tools() {
   local missing=0
-  for command_name in aws curl git ip iptables jq openssl; do
+  for command_name in aws curl git ip iptables jq openssl sha256sum tar; do
     if ! command -v "$command_name" >/dev/null 2>&1; then
       missing=1
     fi
@@ -65,7 +87,25 @@ install_guest_tools() {
   log "installing missing Linux lab tools"
   export DEBIAN_FRONTEND=noninteractive
   apt-get update -qq
-  apt-get install -y -qq awscli curl e2fsprogs git iproute2 iptables jq openssl
+  apt-get install -y -qq awscli ca-certificates curl e2fsprogs file git iproute2 iptables jq openssl tar
+}
+
+prepare_direct_launcher_path() {
+  local command_name command_path
+  mkdir -p "$DIRECT_BIN_DIR"
+  # Lima's Ubuntu guest has systemd, so the agent would otherwise select
+  # systemd-run. Its transient-unit PID exposes systemd-executor through
+  # /proc/<pid>/exe, which cannot satisfy Firecracker ownership checks. Keep
+  # the agent's PATH deliberately small so chooseLauncher selects the direct
+  # process launcher while retaining the host tools used by networking and
+  # local-volume management.
+  for command_name in \
+    e2fsck ip iptables mount mountpoint mkfs.ext4 resize2fs sh sysctl \
+    tune2fs umount; do
+    command_path="$(command -v "$command_name" || true)"
+    [[ -n "$command_path" ]] || die "missing direct-launcher dependency: $command_name"
+    ln -sf "$command_path" "$DIRECT_BIN_DIR/$command_name"
+  done
 }
 
 write_file() {
@@ -81,133 +121,109 @@ cleanup_rootfs_mount() {
   fi
 }
 
-prepare_rootfs() {
-  local source_rootfs="${FIREWORK_E2E_ROOTFS:-}"
-  local source_kernel="${FIREWORK_E2E_KERNEL:-}"
-  local source_firecracker="${FIREWORK_E2E_FIRECRACKER_BIN:-}"
-  [[ -r "$source_rootfs" ]] || die "rootfs is not readable in the lab: $source_rootfs"
-  [[ -r "$source_kernel" ]] || die "kernel is not readable in the lab: $source_kernel"
-  [[ -x "$source_firecracker" ]] || die "Firecracker is not executable in the lab: $source_firecracker"
-
-  cp "$source_rootfs" "$GUEST_ROOTFS"
-  cp "$source_kernel" "$KERNEL"
-  cp "$source_firecracker" "$FIRECRACKER"
-  chmod 755 "$FIRECRACKER"
-
-  mount -o loop "$GUEST_ROOTFS" "$ROOTFS_MOUNT"
-  local busybox=""
-  for candidate in /bin/busybox /sbin/busybox /usr/bin/busybox; do
-    if [[ -x "$ROOTFS_MOUNT$candidate" ]]; then
-      busybox="$candidate"
-      break
-    fi
-  done
-  [[ -n "$busybox" ]] || die "rootfs must contain a static BusyBox binary"
-  [[ -x "$ROOTFS_MOUNT/bin/sh" || -x "$ROOTFS_MOUNT/bin/busybox" ]] || die "rootfs must provide /bin/sh"
-
-  install -D -m 0755 "${BIN_DIR}/fc-init-linux-$BIN_ARCH" "$ROOTFS_MOUNT/sbin/fc-init"
-  rm -f "$ROOTFS_MOUNT/sbin/init"
-  write_file "$ROOTFS_MOUNT/sbin/init" <<'EOF'
-#!/bin/sh
-set -eu
-busybox=/bin/busybox
-if [ ! -x "$busybox" ]; then busybox=/sbin/busybox; fi
-if [ ! -x "$busybox" ]; then busybox=/usr/bin/busybox; fi
-role="${FIREWORK_ROLE:-responder}"
-case "$role" in
-  responder) port=8080 ;;
-  caller)
-    target="${RESPONDER_URL:-}"
-    [ -n "$target" ] || exit 20
-    ready=0
-    attempt=0
-    while [ "$attempt" -lt 30 ]; do
-      if "$busybox" wget -q -O - "$target/health" >/dev/null 2>&1; then
-        ready=1
-        break
-      fi
-      attempt=$((attempt + 1))
-      sleep 1
-    done
-    [ "$ready" -eq 1 ] || exit 21
-    echo cross-node-ok > /tmp/cross-node-result
-    port=8081
-    ;;
-  *) exit 22 ;;
-esac
-exec "$busybox" httpd -f -p "$port" -h /www
-EOF
-  chmod 755 "$ROOTFS_MOUNT/sbin/init"
-  mkdir -p "$ROOTFS_MOUNT/www"
-  printf 'ok\n' > "$ROOTFS_MOUNT/www/health"
-  cleanup_rootfs_mount
+prepare_assets() {
+  [[ -x "$ASSET_SCRIPT" ]] || die "asset preparation script is missing: $ASSET_SCRIPT"
+  FIREWORK_E2E_FIRECRACKER_BIN="$FIRECRACKER" FIREWORK_E2E_KERNEL="$KERNEL" \
+    FIREWORK_E2E_FIRECRACKER_VERSION="${FIREWORK_E2E_FIRECRACKER_VERSION:-1.12.0}" \
+    FIREWORK_E2E_KERNEL_KEY="${FIREWORK_E2E_KERNEL_KEY:-firecracker-ci/v1.12/aarch64/vmlinux-5.10.233}" \
+    bash "$ASSET_SCRIPT" "$RUN_DIR"
+  [[ -x "$FIRECRACKER" ]] || die "prepared Firecracker binary is missing"
+  [[ -r "$KERNEL" ]] || die "prepared kernel is missing"
 }
 
 setup_git_repo() {
   mkdir -p "$CONFIG_DIR/repo/services"
   write_file "$CONFIG_DIR/repo/defaults.yaml" <<EOF
 kernel: "$KERNEL"
-vcpus: 1
-memory_mb: 128
-kernel_args: "console=ttyS0 reboot=k panic=1 pci=off init=/sbin/fc-init"
+vcpus: $ES_VCPUS
+memory_mb: $ES_MEMORY_MB
+kernel_args: "$KERNEL_BOOT_ARGS init=/sbin/fc-init"
 EOF
-  write_file "$CONFIG_DIR/repo/services/caller.yaml" <<EOF
-name: "caller"
-image: "$GUEST_ROOTFS"
+  write_stateful_services
+  git -C "$CONFIG_DIR/repo" init -b main >/dev/null 2>&1 || {
+    git -C "$CONFIG_DIR/repo" init >/dev/null
+    git -C "$CONFIG_DIR/repo" checkout -b main >/dev/null
+  }
+  git -C "$CONFIG_DIR/repo" config user.name firework-local-e2e
+  git -C "$CONFIG_DIR/repo" config user.email firework-local-e2e@localhost
+  git -C "$CONFIG_DIR/repo" add .
+  git -C "$CONFIG_DIR/repo" commit -m "local e2e stateful workload" >/dev/null
+}
+
+write_stateful_services() {
+  local anti_affinity="${1:-}"
+  local volume=""
+  if [[ "${FIREWORK_E2E_ENABLE_VOLUME:-1}" == 1 ]]; then
+    volume="$(printf 'volumes:\n  - name: data\n    type: local\n    mount_path: /usr/share/elasticsearch/data\n    size: %s\n' "$VOLUME_SIZE")"
+  fi
+  write_file "$CONFIG_DIR/repo/services/elasticsearch.yaml" <<EOF
+name: "tenant-2-elasticsearch"
+image: "$IMAGE_DIR/$ES_IMAGE_KEY"
 kernel: "$KERNEL"
-node_type: "general"
+node_type: "stateful"
+vcpus: $ES_VCPUS
+memory_mb: $ES_MEMORY_MB
+kernel_args: "$KERNEL_BOOT_ARGS init=/sbin/fc-init /bin/tini -- /usr/local/bin/docker-entrypoint.sh eswrapper"
 network: true
 port_forwards:
-  - host_port: 18081
-    vm_port: 8081
+  - host_port: 19200
+    vm_port: 9200
 health_check:
   type: "http"
-  port: 8081
-  path: "/health"
-  interval: "2s"
-  timeout: "1s"
-  retries: 5
+  port: 9200
+  path: "/_cluster/health"
+  interval: "15s"
+  timeout: "10s"
+  retries: $HEALTH_RETRIES
 env:
-  FIREWORK_ROLE: "caller"
-cross_node_links:
-  - service: "responder"
-    env: "RESPONDER_URL"
-    host_port: 18080
-    protocol: "http"
-anti_affinity_group: "local-e2e"
+  TENANT_ID: "local-stateful"
+  ES_JAVA_OPTS: "$ES_JAVA_OPTS"
+$volume$anti_affinity
 EOF
-  write_file "$CONFIG_DIR/repo/services/responder.yaml" <<EOF
-name: "responder"
-image: "$GUEST_ROOTFS"
+  write_file "$CONFIG_DIR/repo/services/kibana.yaml" <<EOF
+name: "tenant-2-kibana"
+image: "$IMAGE_DIR/$KIBANA_IMAGE_KEY"
 kernel: "$KERNEL"
-node_type: "general"
+node_type: "stateful"
+vcpus: $KIBANA_VCPUS
+memory_mb: $KIBANA_MEMORY_MB
+kernel_args: "$KERNEL_BOOT_ARGS init=/sbin/fc-init /bin/tini -- /usr/local/bin/kibana-docker"
 network: true
 port_forwards:
-  - host_port: 18080
-    vm_port: 8080
+  - host_port: 15612
+    vm_port: 5601
 health_check:
   type: "http"
-  port: 8080
-  path: "/health"
-  interval: "2s"
-  timeout: "1s"
-  retries: 5
+  port: 5601
+  path: "/api/status"
+  interval: "30s"
+  timeout: "10s"
+  retries: $HEALTH_RETRIES
 env:
-  FIREWORK_ROLE: "responder"
-anti_affinity_group: "local-e2e"
+  TENANT_ID: "local-stateful"
+cross_node_links:
+  - service: "tenant-2-elasticsearch"
+    env: "ELASTICSEARCH_HOSTS"
+    host_port: 19200
+    protocol: "http"
+$anti_affinity
 EOF
-  git -C "$CONFIG_DIR/repo" init -b main >/dev/null 2>&1 || {
-    git -C "$CONFIG_DIR/repo" init >/dev/null
-    git -C "$CONFIG_DIR/repo" checkout -b main >/dev/null
-  }
-  git -C "$CONFIG_DIR/repo" config user.name firework-local-e2e
-  git -C "$CONFIG_DIR/repo" config user.email firework-local-e2e@localhost
-  git -C "$CONFIG_DIR/repo" add .
-  git -C "$CONFIG_DIR/repo" commit -m "local e2e workload" >/dev/null
+}
+
+commit_stateful_update() {
+  local message="$1"
+  git -C "$CONFIG_DIR/repo" add services
+  git -C "$CONFIG_DIR/repo" commit -m "$message" >/dev/null
 }
 
 setup_network() {
-  local namespace ip_address uplink
+  local namespace ip_address uplink resolver_source
+  resolver_source="/run/systemd/resolve/resolv.conf"
+  if [[ ! -r "$resolver_source" ]]; then
+    resolver_source="$RUN_DIR/resolv.conf"
+    cp /etc/resolv.conf "$resolver_source"
+    sed -i 's/127\.0\.0\.53/192.168.5.2/g' "$resolver_source"
+  fi
   ip link add name fw-e2e-br type bridge
   ip addr add 10.254.0.1/24 dev fw-e2e-br
   ip link set fw-e2e-br up
@@ -225,10 +241,21 @@ setup_network() {
     ip -n "$namespace" link set "$uplink" up
     ip -n "$namespace" addr add "$ip_address/24" dev "$uplink"
     ip -n "$namespace" route add default via 10.254.0.1
+    mkdir -p "/etc/netns/$namespace"
+    cp "$resolver_source" "/etc/netns/$namespace/resolv.conf"
     ip netns exec "$namespace" sysctl -w net.ipv4.ip_forward=1 >/dev/null
     ip netns exec "$namespace" sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null
     ip netns exec "$namespace" sysctl -w net.ipv4.conf.default.rp_filter=0 >/dev/null
   done
+  mkdir -p "$RUN_DIR/storage/node-a" "$RUN_DIR/storage/node-b"
+  storage_mount_size="${STORAGE_CAPACITY//i/}"
+  mount -t tmpfs -o "size=$storage_mount_size" firework-e2e-storage-a "$RUN_DIR/storage/node-a"
+  mount -t tmpfs -o "size=$storage_mount_size" firework-e2e-storage-b "$RUN_DIR/storage/node-b"
+  ROOT_OUT_INTERFACE="$(ip -o route show default | awk 'NR == 1 {print $5}')"
+  [[ -n "$ROOT_OUT_INTERFACE" ]] || die "could not determine the Lima guest default interface"
+  iptables -t nat -A POSTROUTING -s 10.254.0.0/24 -o "$ROOT_OUT_INTERFACE" -j MASQUERADE
+  iptables -A FORWARD -s 10.254.0.0/24 -j ACCEPT
+  iptables -A FORWARD -d 10.254.0.0/24 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
   ip route add 172.16.1.0/24 via 10.254.0.11 dev fw-e2e-br
   ip route add 172.16.2.0/24 via 10.254.0.12 dev fw-e2e-br
 }
@@ -290,6 +317,7 @@ store_type: "s3"
 s3_bucket: "$E2E_BUCKET"
 s3_prefix: "cp/v1/"
 s3_region: "$AWS_REGION_VALUE"
+s3_images_bucket: "$IMAGES_BUCKET"
 poll_interval: "2s"
 firecracker_bin: "$FIRECRACKER"
 state_dir: "$STATE_DIR/$node"
@@ -298,7 +326,7 @@ log_level: "debug"
 api_listen_addr: "$namespace_ip:$api_port"
 enable_health_checks: true
 enable_network_setup: true
-enable_capacity_check: false
+enable_capacity_check: true
 vm_subnet: "$vm_subnet"
 vm_gateway: "$vm_gateway"
 vm_bridge: "br-fw-$node"
@@ -310,11 +338,15 @@ registry_key_file: "$STATE_DIR/$node/node.key"
 registry_ca_file: "$PKI_DIR/ca.crt"
 registry_bootstrap_token: "$token"
 registry_heartbeat_interval: "2s"
+storage:
+  local:
+    path: "$RUN_DIR/storage/$node"
+    capacity: "$STORAGE_CAPACITY"
 EOF
 }
 
 wait_http() {
-  local url="$1" ca_file="$2" auth_header="${3:-}" deadline=$((SECONDS + ${FIREWORK_E2E_TIMEOUT:-600}))
+  local url="$1" ca_file="$2" auth_header="${3:-}" deadline=$((SECONDS + E2E_TIMEOUT))
   while (( SECONDS < deadline )); do
     if [[ -n "$auth_header" ]]; then
       if curl --silent --show-error --fail --cacert "$ca_file" -H "$auth_header" "$url" >/dev/null 2>&1; then return 0; fi
@@ -328,6 +360,140 @@ wait_http() {
   return 1
 }
 
+api_get() {
+  local path="$1"
+  curl --silent --show-error --fail --cacert "$PKI_DIR/ca.crt" \
+    -H "Authorization: Bearer $OPERATOR_TOKEN" "https://127.0.0.1:9445$path"
+}
+
+wait_nodes() {
+  local count="$1" deadline=$((SECONDS + E2E_TIMEOUT)) nodes
+  while (( SECONDS < deadline )); do
+    nodes="$(api_get /v1/nodes 2>/dev/null || true)"
+    if jq -e --argjson count "$count" '.count == $count and all(.items[]; .state == "ready")' <<<"$nodes" >/dev/null 2>&1; then
+      printf '%s\n' "$nodes"
+      return 0
+    fi
+    sleep 2
+  done
+  printf '%s\n' "${nodes:-{}}" > "$RUN_DIR/nodes-timeout.json"
+  return 1
+}
+
+wait_services_healthy() {
+  local expected="$1" distinct_nodes="$2" deadline=$((SECONDS + E2E_TIMEOUT)) services
+  while (( SECONDS < deadline )); do
+    services="$(api_get /v1/services 2>/dev/null || true)"
+    if jq -e --argjson expected "$expected" --argjson distinct "$distinct_nodes" \
+      '.count == $expected and all(.items[]; .state == "running" and .health == "healthy") and (([.items[].node] | unique | length) == $distinct)' \
+      <<<"$services" >/dev/null 2>&1; then
+      printf '%s\n' "$services"
+      return 0
+    fi
+    sleep 2
+  done
+  printf '%s\n' "${services:-{}}" > "$RUN_DIR/services-timeout.json"
+  return 1
+}
+
+wait_empty_state() {
+  local deadline=$((SECONDS + E2E_TIMEOUT)) services status
+  while (( SECONDS < deadline )); do
+    services="$(api_get /v1/services 2>/dev/null || true)"
+    status="$(curl --silent --show-error --fail http://10.254.0.11:18081/status 2>/dev/null || true)"
+    if jq -e '.count == 0' <<<"$services" >/dev/null 2>&1 && \
+      jq -e '.desired_services == 0 and (.services | length == 0)' <<<"$status" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 2
+  done
+  return 1
+}
+
+service_node() {
+  local name="$1" services
+  services="$(api_get /v1/services)"
+  jq -r --arg name "$name" '.items[] | select(.name == $name) | .node' <<<"$services"
+}
+
+node_host() {
+  case "$1" in
+    node-a) printf '10.254.0.11\n' ;;
+    node-b) printf '10.254.0.12\n' ;;
+    *) return 1 ;;
+  esac
+}
+
+service_url() {
+  local name="$1" port="$2" node
+  node="$(service_node "$name")"
+  printf 'http://%s:%s\n' "$(node_host "$node")" "$port"
+}
+
+wait_service_http() {
+  local url="$1" deadline=$((SECONDS + E2E_TIMEOUT))
+  while (( SECONDS < deadline )); do
+    if curl --silent --show-error --fail "$url" >/dev/null 2>&1; then return 0; fi
+    sleep 2
+  done
+  return 1
+}
+
+assert_elasticsearch_green() {
+  local endpoint="$1" response
+  wait_service_http "$endpoint/_cluster/health" || die "Elasticsearch endpoint did not become reachable: $endpoint"
+  curl --silent --show-error --fail -X PUT "$endpoint/_settings" \
+    -H 'Content-Type: application/json' \
+    --data '{"index":{"number_of_replicas":0}}' >/dev/null
+  local deadline=$((SECONDS + E2E_TIMEOUT))
+  while (( SECONDS < deadline )); do
+    response="$(curl --silent --show-error --fail "$endpoint/_cluster/health" 2>/dev/null || true)"
+    if jq -e '.status == "green"' <<<"$response" >/dev/null 2>&1; then
+      printf '%s\n' "$response" > "$RUN_DIR/elasticsearch-health.json"
+      return 0
+    fi
+    sleep 5
+  done
+  printf '%s\n' "${response:-{}}" > "$RUN_DIR/elasticsearch-health-timeout.json"
+  return 1
+}
+
+send_webhook() {
+  local delivery_id="$1" payload signature
+  payload="$(jq -cn --arg ref refs/heads/main --arg url "file://$CONFIG_DIR/repo" \
+    '{ref:$ref,repository:{clone_url:$url}}')"
+  signature="$(printf '%s' "$payload" | openssl dgst -sha256 -hmac "$WEBHOOK_SECRET" -binary | od -An -vtx1 | tr -d ' \n')"
+  curl --silent --show-error --fail --cacert "$PKI_DIR/ca.crt" \
+    --resolve controlplane.local:9444:10.254.0.1 \
+    -H 'Content-Type: application/json' \
+    -H 'X-GitHub-Event: push' \
+    -H "X-GitHub-Delivery: $delivery_id" \
+    -H "X-Hub-Signature-256: sha256=$signature" \
+    --data "$payload" https://controlplane.local:9444/v1/events/github >/dev/null
+}
+
+start_agent() {
+  local node="$1"
+  PATH="$DIRECT_BIN_DIR" ip netns exec "fw-e2e-$node" "$BIN_DIR/firework-agent-linux-$BIN_ARCH" \
+    --config "$CONFIG_DIR/agent-$node.yaml" > "$LOG_DIR/$node.log" 2>&1 &
+  AGENT_PIDS+=("$!")
+}
+
+stop_agent() {
+  local index="$1"
+  local pid="${AGENT_PIDS[$index]:-}"
+  [[ -n "$pid" ]] || return 0
+  kill -TERM "$pid" >/dev/null 2>&1 || true
+  wait "$pid" >/dev/null 2>&1 || true
+  AGENT_PIDS[index]=""
+}
+
+commit_and_notify() {
+  local message="$1" delivery="$2"
+  commit_stateful_update "$message"
+  send_webhook "$delivery"
+}
+
 stop_processes() {
   local pid
   for pid in "${AGENT_PIDS[@]}" "$CONTROLPLANE_PID"; do
@@ -347,6 +513,14 @@ delete_network() {
   ip route del 172.16.2.0/24 via 10.254.0.12 dev fw-e2e-br >/dev/null 2>&1 || true
   ip netns del fw-e2e-node-a >/dev/null 2>&1 || true
   ip netns del fw-e2e-node-b >/dev/null 2>&1 || true
+  rm -rf /etc/netns/fw-e2e-node-a /etc/netns/fw-e2e-node-b
+  umount "$RUN_DIR/storage/node-a" >/dev/null 2>&1 || true
+  umount "$RUN_DIR/storage/node-b" >/dev/null 2>&1 || true
+  if [[ -n "$ROOT_OUT_INTERFACE" ]]; then
+    iptables -t nat -D POSTROUTING -s 10.254.0.0/24 -o "$ROOT_OUT_INTERFACE" -j MASQUERADE >/dev/null 2>&1 || true
+    iptables -D FORWARD -s 10.254.0.0/24 -j ACCEPT >/dev/null 2>&1 || true
+    iptables -D FORWARD -d 10.254.0.0/24 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT >/dev/null 2>&1 || true
+  fi
   ip link del fw-e2e-br >/dev/null 2>&1 || true
 }
 
@@ -359,11 +533,14 @@ write_manifest() {
     --arg mode "$RUN_MODE" \
     --arg commit "$FIREWORK_COMMIT" \
     --arg bucket "$E2E_BUCKET" \
+    --arg images_bucket "$IMAGES_BUCKET" \
+    --arg es_image "$ES_IMAGE_KEY" \
+    --arg kibana_image "$KIBANA_IMAGE_KEY" \
     --arg region "$AWS_REGION_VALUE" \
     --arg run_dir "$RUN_DIR" \
     --arg controlplane_pid "${CONTROLPLANE_PID:-}" \
     --argjson agent_pids "$agent_pids_json" \
-    '{status:$status,mode:$mode,firework_commit:$commit,bucket:$bucket,region:$region,run_dir:$run_dir,controlplane_pid:$controlplane_pid,agent_pids:$agent_pids}' \
+    '{status:$status,mode:$mode,scenario:"stateful",firework_commit:$commit,bucket:$bucket,images_bucket:$images_bucket,images:{elasticsearch:$es_image,kibana:$kibana_image},region:$region,run_dir:$run_dir,controlplane_pid:$controlplane_pid,agent_pids:$agent_pids}' \
     > "$RUN_DIR/manifest.json"
 }
 
@@ -398,10 +575,11 @@ for command_name in mountpoint mount umount install; do require_cmd "$command_na
 [[ -e /dev/net/tun ]] || die "/dev/net/tun is required in the lab"
 
 log "preparing Firecracker assets"
-prepare_rootfs
+prepare_assets
 "$SCRIPT_DIR/generate-pki.sh" "$PKI_DIR"
 setup_git_repo
 setup_network
+prepare_direct_launcher_path
 
 OPERATOR_TOKEN="local-e2e-operator-$FIREWORK_COMMIT"
 write_controlplane_config "$OPERATOR_TOKEN"
@@ -413,44 +591,84 @@ log "starting combined control plane"
 CONTROLPLANE_PID=$!
 wait_http "https://127.0.0.1:9445/healthz" "$PKI_DIR/ca.crt" || die "control-plane API did not become healthy"
 
-log "starting two isolated agents"
-ip netns exec fw-e2e-node-a "$BIN_DIR/firework-agent-linux-$BIN_ARCH" --config "$CONFIG_DIR/agent-node-a.yaml" > "$LOG_DIR/node-a.log" 2>&1 &
-AGENT_PIDS+=("$!")
-ip netns exec fw-e2e-node-b "$BIN_DIR/firework-agent-linux-$BIN_ARCH" --config "$CONFIG_DIR/agent-node-b.yaml" > "$LOG_DIR/node-b.log" 2>&1 &
-AGENT_PIDS+=("$!")
+log "starting node-a and proving the colocated stateful topology"
+start_agent node-a
 wait_http "http://10.254.0.11:18081/healthz" "" || die "node-a API did not become healthy"
+nodes_json="$(wait_nodes 1)" || die "node-a did not become ready"
+services_json="$(wait_services_healthy 2 1)" || die "stateful services did not converge in the colocated topology"
+printf '%s\n' "$nodes_json" > "$RUN_DIR/nodes-colocated.json"
+printf '%s\n' "$services_json" > "$RUN_DIR/services-colocated.json"
+es_endpoint="$(service_url tenant-2-elasticsearch 19200)"
+kibana_endpoint="$(service_url tenant-2-kibana 15612)"
+assert_elasticsearch_green "$es_endpoint" || die "Elasticsearch did not become green in the colocated topology"
+wait_service_http "$kibana_endpoint/api/status" || die "Kibana did not become reachable in the colocated topology"
+[[ -s "$IMAGE_DIR/$ES_IMAGE_KEY" && -s "$IMAGE_DIR/$KIBANA_IMAGE_KEY" ]] \
+  || die "the agent did not download both real S3 workload images"
+
+log "starting node-b and checking that the colocated assignment remains stable"
+start_agent node-b
 wait_http "http://10.254.0.12:18082/healthz" "" || die "node-b API did not become healthy"
-
-nodes_url="https://127.0.0.1:9445/v1/nodes"
-services_url="https://127.0.0.1:9445/v1/services"
-auth_header="Authorization: Bearer $OPERATOR_TOKEN"
-deadline=$((SECONDS + ${FIREWORK_E2E_TIMEOUT:-600}))
-nodes_json=""
-services_json=""
-while (( SECONDS < deadline )); do
-  nodes_json="$(curl --silent --show-error --fail --cacert "$PKI_DIR/ca.crt" -H "$auth_header" "$nodes_url" 2>/dev/null || true)"
-  services_json="$(curl --silent --show-error --fail --cacert "$PKI_DIR/ca.crt" -H "$auth_header" "$services_url" 2>/dev/null || true)"
-  if jq -e '.count == 2 and ([.items[].node_id] | unique | length == 2) and all(.items[]; .state == "ready")' <<<"$nodes_json" >/dev/null 2>&1 \
-    && jq -e '.count == 2 and all(.items[]; .state == "running" and .health == "healthy") and ([.items[].node] | unique | length == 2)' <<<"$services_json" >/dev/null 2>&1; then
+nodes_json="$(wait_nodes 2)" || die "two nodes did not become ready"
+services_json="$(wait_services_healthy 2 1)" || die "services unexpectedly failed while node-b enrolled"
+printf '%s\n' "$nodes_json" > "$RUN_DIR/nodes-two-ready.json"
+printf '%s\n' "$services_json" > "$RUN_DIR/services-colocated-two-nodes.json"
+
+log "updating the same stateful workload with anti-affinity and proving movement"
+write_stateful_services 'anti_affinity_group: "stateful"'
+commit_and_notify "split stateful services across nodes" "split-$FIREWORK_COMMIT"
+services_json="$(wait_services_healthy 2 2)" || die "stateful services did not converge after movement"
+printf '%s\n' "$services_json" > "$RUN_DIR/services-split.json"
+es_endpoint="$(service_url tenant-2-elasticsearch 19200)"
+kibana_endpoint="$(service_url tenant-2-kibana 15612)"
+assert_elasticsearch_green "$es_endpoint" || die "Elasticsearch did not become green after movement"
+wait_service_http "$kibana_endpoint/api/status" || die "Kibana did not become reachable after movement"
+
+log "restarting node-a and verifying surviving VM and local-volume adoption"
+marker="local-validation-$(date -u +%Y%m%dT%H%M%SZ)"
+curl --silent --show-error --fail -X PUT "$es_endpoint/local-validation/_doc/marker" \
+  -H 'Content-Type: application/json' --data "{\"marker\":\"$marker\"}" >/dev/null
+stop_agent 0
+start_agent node-a
+wait_http "http://10.254.0.11:18081/healthz" "" || die "node-a did not recover after agent restart"
+services_json="$(wait_services_healthy 2 2)" || die "services did not converge after agent restart"
+printf '%s\n' "$services_json" > "$RUN_DIR/services-after-agent-restart.json"
+curl --silent --show-error --fail "$es_endpoint/local-validation/_doc/marker" \
+  | jq -e --arg marker "$marker" '._source.marker == $marker' >/dev/null 2>&1 \
+  || die "Elasticsearch volume marker was not readable after agent restart"
+
+log "removing the desired state and checking explicit empty assignment convergence"
+rm -f "$CONFIG_DIR/repo/services"/*.yaml
+commit_and_notify "remove stateful workload" "empty-$FIREWORK_COMMIT"
+wait_empty_state || die "agents did not converge to an explicit empty desired state"
+
+log "restoring the stateful workload for stale-node and port ownership checks"
+write_stateful_services 'anti_affinity_group: "stateful"'
+commit_and_notify "restore stateful workload" "restore-$FIREWORK_COMMIT"
+services_json="$(wait_services_healthy 2 2)" || die "stateful workload did not restore after empty state"
+
+log "stopping node-b and asserting stale-node visibility before recovery"
+stop_agent 1
+stale_deadline=$((SECONDS + 90))
+stale_seen=0
+while (( SECONDS < stale_deadline )); do
+  nodes_json="$(api_get /v1/nodes 2>/dev/null || true)"
+  if jq -e '.items[] | select(.node_id == "node-b" and .state == "down")' <<<"$nodes_json" >/dev/null 2>&1; then
+    stale_seen=1
     break
   fi
   sleep 2
 done
-
-jq -e '.count == 2 and ([.items[].node_id] | unique | length == 2)' <<<"$nodes_json" >/dev/null \
-  || die "two nodes did not register: $nodes_json"
-jq -e '.count == 2 and all(.items[]; .state == "running" and .health == "healthy") and ([.items[].node] | unique | length == 2)' <<<"$services_json" >/dev/null \
-  || die "two healthy cross-node services did not converge: $services_json"
-
-caller_node="$(jq -r '.items[] | select(.name == "caller") | .node' <<<"$services_json")"
-case "$caller_node" in
-  node-a) caller_endpoint="http://10.254.0.11:18081" ;;
-  node-b) caller_endpoint="http://10.254.0.12:18082" ;;
-  *) die "caller service has no valid node placement: $caller_node" ;;
-esac
-curl --silent --show-error --fail "$caller_endpoint/health" >/dev/null \
-  || die "caller endpoint did not become reachable; cross-node link likely failed"
-
+[[ "$stale_seen" == 1 ]] || die "node-b did not become visibly down after heartbeat loss"
+start_agent node-b
+wait_http "http://10.254.0.12:18082/healthz" "" || die "node-b did not recover after stale-node check"
+services_json="$(wait_services_healthy 2 2)" || die "services did not reconverge after stale-node recovery"
+
+log "checking per-node host-port ownership and rendered S3 state"
+es_node="$(service_node tenant-2-elasticsearch)"
+kibana_node="$(service_node tenant-2-kibana)"
+[[ "$es_node" != "$kibana_node" ]] || die "anti-affinity placement regressed: both services are on $es_node"
+jq -e '[.items[].port_forwards[]?.host_port] as $ports | ($ports | length) == ($ports | unique | length)' <<<"$services_json" >/dev/null \
+  || die "the final state contains duplicate host-port ownership"
 for node in node-a node-b; do
   aws s3api head-object --bucket "$E2E_BUCKET" --key "cp/v1/nodes/$node.yaml" >/dev/null \
     || die "real S3 bucket has no rendered config for $node"
@@ -458,6 +676,7 @@ done
 aws s3api list-objects-v2 --bucket "$E2E_BUCKET" --prefix cp/v1/ --output json > "$RUN_DIR/s3-inventory.json"
 printf '%s\n' "$nodes_json" > "$RUN_DIR/nodes.json"
 printf '%s\n' "$services_json" > "$RUN_DIR/services.json"
+cp "$RUN_DIR/assets-manifest.json" "$RUN_DIR/asset-provenance.json"
 SCENARIO_STATUS="passed"
-log "local two-node E2E passed"
-log "node placement: $(jq -r '.items[] | [.name, .node] | @tsv' <<<"$services_json" | tr '\n' ' ')"
+log "local stateful two-node E2E passed"
+log "final node placement: $(jq -r '.items[] | [.name, .node] | @tsv' <<<"$services_json" | tr '\n' ' ')"
diff --git a/validation/e2e-local/scripts/validate.sh b/validation/e2e-local/scripts/validate.sh
index 01960bb..1aaabba 100755
--- a/validation/e2e-local/scripts/validate.sh
+++ b/validation/e2e-local/scripts/validate.sh
@@ -25,8 +25,9 @@ require_cmd go
 require_cmd jq
 require_cmd make
 
-if [[ -z "${AWS_ACCESS_KEY_ID:-}" && -n "${AWS_PROFILE:-}" ]]; then
-  eval "$(aws configure export-credentials --profile "$AWS_PROFILE" --format env)"
+if [[ -z "${AWS_ACCESS_KEY_ID:-}" ]]; then
+  aws_profile="${AWS_PROFILE:-${FIREWORK_E2E_AWS_PROFILE:-artemnikitin}}"
+  eval "$(aws configure export-credentials --profile "$aws_profile" --format env)"
   unset AWS_PROFILE
 fi
 
@@ -35,6 +36,10 @@ export AWS_DEFAULT_REGION="$AWS_REGION"
 [[ -n "$AWS_REGION" ]] || die "AWS_REGION or AWS_DEFAULT_REGION is required"
 aws sts get-caller-identity >/dev/null || die "AWS credentials are not usable"
 
+images_bucket="${FIREWORK_E2E_IMAGES_BUCKET:-artemnikitin-firework-images}"
+[[ "$images_bucket" =~ ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ ]] \
+  || die "FIREWORK_E2E_IMAGES_BUCKET is not a valid S3 bucket name"
+
 account_id="$(aws sts get-caller-identity --query Account --output text)"
 run_id="$(date -u +%Y%m%dt%H%M%S)-$$"
 bucket="firework-local-e2e-${account_id}-${run_id//[^a-zA-Z0-9-]/-}"
@@ -64,9 +69,17 @@ if [[ "$mode" == lima && "$(uname -s)" != Darwin ]]; then
   die "FIREWORK_E2E_MODE=lima requires macOS"
 fi
 
-[[ -n "${FIREWORK_E2E_FIRECRACKER_BIN:-}" && -x "$FIREWORK_E2E_FIRECRACKER_BIN" ]] || die "set executable FIREWORK_E2E_FIRECRACKER_BIN"
-[[ -n "${FIREWORK_E2E_KERNEL:-}" && -r "$FIREWORK_E2E_KERNEL" ]] || die "set readable FIREWORK_E2E_KERNEL"
-[[ -n "${FIREWORK_E2E_ROOTFS:-}" && -r "$FIREWORK_E2E_ROOTFS" ]] || die "set readable FIREWORK_E2E_ROOTFS"
+if [[ "$mode" == lima ]]; then
+  [[ "$(uname -m)" == arm64 ]] || die "Lima mode requires an Apple Silicon host"
+  require_cmd limactl
+  log "host capability probe: macOS $(sw_vers -productVersion), Lima $(limactl --version | head -n 1)"
+  log "the guest probe will verify /dev/kvm, /dev/net/tun, namespaces, bridges, and iptables"
+fi
+
+aws s3api head-object --bucket "$images_bucket" --key "${FIREWORK_E2E_ES_IMAGE_KEY:-tenant-2-elasticsearch-rootfs.ext4}" >/dev/null \
+  || die "cannot read the Elasticsearch rootfs from s3://$images_bucket (set FIREWORK_E2E_IMAGES_BUCKET or image key overrides)"
+aws s3api head-object --bucket "$images_bucket" --key "${FIREWORK_E2E_KIBANA_IMAGE_KEY:-tenant-2-kibana-rootfs.ext4}" >/dev/null \
+  || die "cannot read the Kibana rootfs from s3://$images_bucket (set FIREWORK_E2E_IMAGES_BUCKET or image key overrides)"
 
 log "building Linux Firework binaries"
 (cd "$REPO_ROOT" && make "build-linux-$host_arch" >/dev/null)
@@ -79,6 +92,33 @@ else
 fi
 mkdir -p "$host_run_dir"
 
+conditional_key="probe/conditional-write-$run_id"
+conditional_body="$host_run_dir/conditional-write.txt"
+conditional_updated_body="$host_run_dir/conditional-write-updated.txt"
+printf 'local-e2e-conditional-write\n' > "$conditional_body"
+printf 'local-e2e-conditional-write-updated\n' > "$conditional_updated_body"
+
+run_conditional_write_probe() {
+  local first_etag second_etag
+  aws s3api put-object --bucket "$bucket" --key "$conditional_key" \
+    --body "$conditional_body" --if-none-match '*' >/dev/null
+  if aws s3api put-object --bucket "$bucket" --key "$conditional_key" \
+    --body "$conditional_body" --if-none-match '*' >/dev/null 2>&1; then
+    die "S3 If-None-Match conditional write unexpectedly overwrote an object"
+  fi
+  first_etag="$(aws s3api head-object --bucket "$bucket" --key "$conditional_key" --query ETag --output text)"
+  aws s3api put-object --bucket "$bucket" --key "$conditional_key" \
+    --body "$conditional_updated_body" --if-match "$first_etag" >/dev/null
+  second_etag="$(aws s3api head-object --bucket "$bucket" --key "$conditional_key" --query ETag --output text)"
+  if aws s3api put-object --bucket "$bucket" --key "$conditional_key" \
+    --body "$conditional_body" --if-match "$first_etag" >/dev/null 2>&1; then
+    die "S3 stale If-Match conditional write unexpectedly succeeded"
+  fi
+  jq -n --arg key "$conditional_key" --arg first "$first_etag" --arg second "$second_etag" \
+    '{key:$key,if_none_match:"passed",if_match:"passed",stale_if_match:"passed",first_etag:$first,second_etag:$second}' \
+    > "$host_run_dir/conditional-write.json"
+}
+
 instance=""
 guest_run_dir="/tmp/firework-e2e-local-$run_id"
 status=1
@@ -139,15 +179,16 @@ else
     --create-bucket-configuration "LocationConstraint=$AWS_REGION" >/dev/null
 fi
 bucket_created=1
+run_conditional_write_probe
 
 if [[ "$mode" == linux ]]; then
   "$SCRIPT_DIR/check-env.sh"
-  sudo_env="AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_REGION,AWS_DEFAULT_REGION,AWS_EC2_METADATA_DISABLED,FIREWORK_E2E_KEEP,FIREWORK_E2E_TIMEOUT,FIREWORK_E2E_FIRECRACKER_BIN,FIREWORK_E2E_KERNEL,FIREWORK_E2E_ROOTFS"
+  sudo_env="AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_REGION,AWS_DEFAULT_REGION,AWS_EC2_METADATA_DISABLED,FIREWORK_E2E_KEEP,FIREWORK_E2E_TIMEOUT,FIREWORK_E2E_HEALTH_RETRIES,FIREWORK_E2E_IMAGES_BUCKET,FIREWORK_E2E_ES_IMAGE_KEY,FIREWORK_E2E_KIBANA_IMAGE_KEY,FIREWORK_E2E_FIRECRACKER_VERSION,FIREWORK_E2E_KERNEL_KEY,FIREWORK_E2E_SERVICE_VCPUS,FIREWORK_E2E_SERVICE_MEMORY_MB,FIREWORK_E2E_ES_VCPUS,FIREWORK_E2E_ES_MEMORY_MB,FIREWORK_E2E_KIBANA_VCPUS,FIREWORK_E2E_KIBANA_MEMORY_MB,FIREWORK_E2E_ES_JAVA_OPTS,FIREWORK_E2E_ENABLE_VOLUME,FIREWORK_E2E_VOLUME_SIZE,FIREWORK_E2E_STORAGE_CAPACITY"
   if [[ "$(id -u)" -eq 0 ]]; then
-    bash "$GUEST_RUNNER" "$host_run_dir" "$bucket" "$AWS_REGION" "$commit" linux
+    bash "$GUEST_RUNNER" "$host_run_dir" "$bucket" "$AWS_REGION" "$commit" linux "$images_bucket"
   else
     require_cmd sudo
-    sudo --preserve-env="$sudo_env" bash "$GUEST_RUNNER" "$host_run_dir" "$bucket" "$AWS_REGION" "$commit" linux
+    sudo --preserve-env="$sudo_env" bash "$GUEST_RUNNER" "$host_run_dir" "$bucket" "$AWS_REGION" "$commit" linux "$images_bucket"
   fi
   status=$?
 else
@@ -156,8 +197,11 @@ else
 
   instance="firework-e2e-$run_id"
   log "starting Lima instance $instance"
+  lima_cpus="${FIREWORK_E2E_LIMA_CPUS:-8}"
+  lima_memory="${FIREWORK_E2E_LIMA_MEMORY_GB:-12}"
+  lima_disk="${FIREWORK_E2E_LIMA_DISK_GB:-60}"
   limactl start --yes --name "$instance" --plain --vm-type=vz --nested-virt \
-    --cpus 4 --memory 8 --disk 30 >/dev/null
+    --cpus "$lima_cpus" --memory "$lima_memory" --disk "$lima_disk" >/dev/null
   limactl shell --yes "$instance" -- sudo -n apt-get update -qq >/dev/null
   limactl shell --yes "$instance" -- sudo -n apt-get install -y -qq awscli curl e2fsprogs git iproute2 iptables jq openssl >/dev/null
 
@@ -165,21 +209,32 @@ else
   limactl shell --yes "$instance" -- mkdir -p "$guest_root/bin" "$guest_root/images" "$guest_root/scripts"
   limactl copy "$GUEST_RUNNER" "$instance:$guest_root/run-guest.sh"
   limactl copy "$SCRIPT_DIR/generate-pki.sh" "$instance:$guest_root/scripts/generate-pki.sh"
+  limactl copy "$SCRIPT_DIR/prepare-assets.sh" "$instance:$guest_root/scripts/prepare-assets.sh"
   limactl copy "$SCRIPT_DIR/collect-diagnostics.sh" "$instance:$guest_root/scripts/collect-diagnostics.sh"
   limactl copy "$REPO_ROOT/bin/firework-agent-linux-arm64" "$instance:$guest_root/bin/firework-agent-linux-arm64"
   limactl copy "$REPO_ROOT/bin/firework-controlplane-linux-arm64" "$instance:$guest_root/bin/firework-controlplane-linux-arm64"
   limactl copy "$REPO_ROOT/bin/fc-init-linux-arm64" "$instance:$guest_root/bin/fc-init-linux-arm64"
-  limactl copy "$FIREWORK_E2E_FIRECRACKER_BIN" "$instance:$guest_root/images/firecracker"
-  limactl copy "$FIREWORK_E2E_KERNEL" "$instance:$guest_root/images/vmlinux-source"
-  limactl copy "$FIREWORK_E2E_ROOTFS" "$instance:$guest_root/images/rootfs-source.ext4"
-  sudo_env="AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_REGION,AWS_DEFAULT_REGION,AWS_EC2_METADATA_DISABLED,FIREWORK_E2E_KEEP,FIREWORK_E2E_TIMEOUT"
-  LIMA_SHELLENV_ALLOW="+AWS_ACCESS_KEY_ID,+AWS_SECRET_ACCESS_KEY,+AWS_SESSION_TOKEN,+AWS_REGION,+AWS_DEFAULT_REGION" \
+  if [[ -n "${FIREWORK_E2E_FIRECRACKER_BIN:-}" ]]; then
+    [[ -x "$FIREWORK_E2E_FIRECRACKER_BIN" ]] || die "FIREWORK_E2E_FIRECRACKER_BIN is not executable"
+    limactl copy "$FIREWORK_E2E_FIRECRACKER_BIN" "$instance:$guest_root/bin/firecracker"
+  fi
+  if [[ -n "${FIREWORK_E2E_KERNEL:-}" ]]; then
+    [[ -r "$FIREWORK_E2E_KERNEL" ]] || die "FIREWORK_E2E_KERNEL is not readable"
+    limactl copy "$FIREWORK_E2E_KERNEL" "$instance:$guest_root/images/vmlinux"
+  fi
+  sudo_env="AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_REGION,AWS_DEFAULT_REGION,AWS_EC2_METADATA_DISABLED,FIREWORK_E2E_KEEP,FIREWORK_E2E_TIMEOUT,FIREWORK_E2E_HEALTH_RETRIES,FIREWORK_E2E_IMAGES_BUCKET,FIREWORK_E2E_ES_IMAGE_KEY,FIREWORK_E2E_KIBANA_IMAGE_KEY,FIREWORK_E2E_FIRECRACKER_VERSION,FIREWORK_E2E_KERNEL_KEY,FIREWORK_E2E_SERVICE_VCPUS,FIREWORK_E2E_SERVICE_MEMORY_MB,FIREWORK_E2E_ES_VCPUS,FIREWORK_E2E_ES_MEMORY_MB,FIREWORK_E2E_KIBANA_VCPUS,FIREWORK_E2E_KIBANA_MEMORY_MB,FIREWORK_E2E_ES_JAVA_OPTS,FIREWORK_E2E_ENABLE_VOLUME,FIREWORK_E2E_VOLUME_SIZE,FIREWORK_E2E_STORAGE_CAPACITY"
+  LIMA_SHELLENV_ALLOW="AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_REGION,AWS_DEFAULT_REGION,AWS_EC2_METADATA_DISABLED,FIREWORK_E2E_KEEP,FIREWORK_E2E_TIMEOUT,FIREWORK_E2E_HEALTH_RETRIES,FIREWORK_E2E_IMAGES_BUCKET,FIREWORK_E2E_ES_IMAGE_KEY,FIREWORK_E2E_KIBANA_IMAGE_KEY,FIREWORK_E2E_FIRECRACKER_VERSION,FIREWORK_E2E_KERNEL_KEY,FIREWORK_E2E_SERVICE_VCPUS,FIREWORK_E2E_SERVICE_MEMORY_MB,FIREWORK_E2E_ES_VCPUS,FIREWORK_E2E_ES_MEMORY_MB,FIREWORK_E2E_KIBANA_VCPUS,FIREWORK_E2E_KIBANA_MEMORY_MB,FIREWORK_E2E_ES_JAVA_OPTS,FIREWORK_E2E_ENABLE_VOLUME,FIREWORK_E2E_VOLUME_SIZE,FIREWORK_E2E_STORAGE_CAPACITY" \
     limactl shell --yes --preserve-env "$instance" -- \
       sudo -n --preserve-env="$sudo_env" env \
         FIREWORK_E2E_FIRECRACKER_BIN="$guest_root/images/firecracker" \
-        FIREWORK_E2E_KERNEL="$guest_root/images/vmlinux-source" \
-        FIREWORK_E2E_ROOTFS="$guest_root/images/rootfs-source.ext4" \
+        FIREWORK_E2E_KERNEL="$guest_root/images/vmlinux" \
+        FIREWORK_E2E_FIRECRACKER_VERSION="${FIREWORK_E2E_FIRECRACKER_VERSION:-1.12.0}" \
+        FIREWORK_E2E_KERNEL_KEY="${FIREWORK_E2E_KERNEL_KEY:-firecracker-ci/v1.12/aarch64/vmlinux-5.10.233}" \
+        FIREWORK_E2E_IMAGES_BUCKET="$images_bucket" \
+        FIREWORK_E2E_ES_IMAGE_KEY="${FIREWORK_E2E_ES_IMAGE_KEY:-tenant-2-elasticsearch-rootfs.ext4}" \
+        FIREWORK_E2E_KIBANA_IMAGE_KEY="${FIREWORK_E2E_KIBANA_IMAGE_KEY:-tenant-2-kibana-rootfs.ext4}" \
         FIREWORK_E2E_SCRIPT_DIR="$guest_root/scripts" \
-        bash "$guest_root/run-guest.sh" "$guest_root" "$bucket" "$AWS_REGION" "$commit" lima
+        FIREWORK_E2E_ASSET_SCRIPT="$guest_root/scripts/prepare-assets.sh" \
+        bash "$guest_root/run-guest.sh" "$guest_root" "$bucket" "$AWS_REGION" "$commit" lima "$images_bucket"
   status=$?
 fi