Implement initial setup for local validation

Makefile

@@ -26,7 +26,7 @@ LDFLAGS   := -s -w \
 	-X '$(MODULE)/internal/version.Commit=$(COMMIT)' \
 	-X '$(MODULE)/internal/version.BuildTime=$(BUILD_TIME)'
 
-.PHONY: all build-all build-agent build-controlplane build-configcheck build-fireworkctl build-fireworkctl-release build-fc-init build-linux-amd64 build-linux-arm64 clean test test-verbose test-race lint vet fmt tidy run smoke-local docker-build-controlplane-image docker-push-controlplane-image push-controlplane-image install help
+.PHONY: all build-all build-agent build-controlplane build-configcheck build-fireworkctl build-fireworkctl-release build-fc-init build-linux-amd64 build-linux-arm64 clean test test-verbose test-race lint vet fmt tidy run smoke-local validate-e2e-local validate-e2e-local-clean docker-build-controlplane-image docker-push-controlplane-image push-controlplane-image install help
 
 all: build-all ## Alias for build-all
 
@@ -118,6 +118,12 @@ run: build-agent ## Build and run with example config
 smoke-local: ## Run local smoke test with fake firecracker
 	./scripts/smoke-local.sh
 
+validate-e2e-local: ## Run the local two-node E2E lab with real Firecracker and S3
+	./validation/e2e-local/scripts/validate.sh
+
+validate-e2e-local-clean: ## Destroy a retained local E2E lab (set FIREWORK_E2E_MANIFEST)
+	./validation/e2e-local/scripts/destroy-lab.sh "$(FIREWORK_E2E_MANIFEST)"
+
 docker-build-controlplane-image: ## Build control-plane image locally (linux/amd64)
 	docker buildx build --platform linux/amd64 --file Dockerfile.controlplane \
 		--build-arg VERSION="$(VERSION)" \

internal/vm/manager.go

@@ -538,6 +538,10 @@ func (m *Manager) writeVMConfig(vmDir string, svc config.ServiceConfig, prepared
 		Drives:            drives,
 		MachineConfig:     firecrackerMachineConfig{VCPUCount: svc.VCPUs, MemSizeMiB: svc.MemoryMB},
 		NetworkInterfaces: networkInterfaces,
+		// The VirtIO-RNG device prevents guests without a usable hardware
+		// random source from blocking application startup on /dev/random.
+		// This matters for arm64 guests nested inside Lima/VZ in particular.
+		Entropy: &firecrackerEntropyDevice{},
 	}
 	configJSON, err := json.MarshalIndent(vmConfig, "", "  ")
 	if err != nil {
@@ -556,6 +560,7 @@ type firecrackerConfig struct {
 	Drives            []firecrackerDrive            `json:"drives"`
 	MachineConfig     firecrackerMachineConfig      `json:"machine-config"`
 	NetworkInterfaces []firecrackerNetworkInterface `json:"network-interfaces,omitempty"`
+	Entropy           *firecrackerEntropyDevice     `json:"entropy,omitempty"`
 }
 
 type firecrackerBootSource struct {
@@ -575,6 +580,8 @@ type firecrackerMachineConfig struct {
 	MemSizeMiB int `json:"mem_size_mib"`
 }
 
+type firecrackerEntropyDevice struct{}
+
 type firecrackerNetworkInterface struct {
 	IfaceID     string `json:"iface_id"`
 	GuestMAC    string `json:"guest_mac"`

internal/vm/manager_test.go

@@ -75,6 +75,9 @@ func TestWriteVMConfigAddsDeterministicVolumeDrivesAndPayload(t *testing.T) {
 	if len(cfg.Drives) != 3 || cfg.Drives[1].PathOnHost != "/a.ext4" || cfg.Drives[2].PathOnHost != "/z.ext4" {
 		t.Fatalf("unexpected drives: %#v", cfg.Drives)
 	}
+	if cfg.Entropy == nil {
+		t.Fatal("expected a VirtIO-RNG entropy device")
+	}
 	fields := strings.Fields(cfg.BootSource.BootArgs)
 	var encoded string
 	for i, field := range fields {

validation/e2e-local/README.md

@@ -0,0 +1,87 @@
+# Local stateful two-node E2E validation
+
+This harness runs one combined Firework setup inside an arm64 Lima Linux
+guest on Apple Silicon:
+
+```text
+one control plane (registry + events + controller + API)
+                         |
+                 real per-run S3 bucket
+                         |
+       node-a namespace ---------------- node-b namespace
+       firework-agent                    firework-agent
+       Firecracker microVMs              Firecracker microVMs
+```
+
+The workload uses the arm64 rootfs images produced by
+`firework-gitops-example`: Elasticsearch and Kibana start together on node-a,
+then the same desired state is changed to anti-affine placement and Kibana
+moves to node-b. Both phases require eventual `running`/`healthy` convergence;
+short startup or movement downtime is expected and allowed. Elasticsearch is
+also driven to cluster status `green` after its single-node replica setting is
+adjusted for this validation.
+
+The run additionally checks the real S3 state/rendered-config path, local
+volume creation and reuse, agent restart/adoption, explicit empty desired
+state, stale-node visibility and recovery, cross-node routing, and final
+per-node port ownership. CI, Linux-native execution, and hosted-KVM probing
+are Milestone 2 work and are not required by this local command today.
+
+## Prerequisites
+
+- Apple Silicon macOS with Lima 2.x and a Lima `vz` guest that exposes
+  readable/writable `/dev/kvm` and `/dev/net/tun`.
+- Go and the repository build toolchain.
+- AWS credentials able to create/list/write/delete a disposable S3 bucket and
+  read the workload-image bucket.
+- Access to the existing arm64 GitOps image bucket. The default is
+  `artemnikitin-firework-images`; override it with
+  `FIREWORK_E2E_IMAGES_BUCKET` when needed.
+
+The harness downloads and verifies Firecracker 1.12.0 arm64 and uses the pinned
+Firecracker CI kernel `firecracker-ci/v1.12/aarch64/vmlinux-5.10.233`. The VM
+configuration enables Firecracker's VirtIO-RNG device so Java/Node workloads
+do not wait indefinitely for guest entropy. Override either asset pin only for
+an intentional compatibility investigation. The two workload rootfs images
+are not copied into the repository or manually modified: the agents download
+them through their production S3 image-sync path.
+
+## Run
+
+```bash
+export AWS_REGION=us-east-1
+export FIREWORK_E2E_AWS_PROFILE=artemnikitin
+make validate-e2e-local
+```
+
+Useful local options:
+
+- `FIREWORK_E2E_KEEP=1` retains the Lima guest, logs, manifest and disposable
+  S3 bucket for inspection. Clean it with
+  `make validate-e2e-local-clean FIREWORK_E2E_MANIFEST=<manifest>`.
+- `FIREWORK_E2E_TIMEOUT=1800` changes the bounded scenario timeout. The default
+  is intentionally generous because these production-sized rootfs images can
+  take several minutes to initialize under nested virtualization.
+- `FIREWORK_E2E_LIMA_CPUS=8`, `FIREWORK_E2E_LIMA_MEMORY_GB=12`, and
+  `FIREWORK_E2E_LIMA_DISK_GB=60` size the local guest.
+- By default Elasticsearch gets 4 vCPUs and 6 GiB, while Kibana gets 2 vCPUs
+  and 4 GiB. Override these with `FIREWORK_E2E_ES_VCPUS`,
+  `FIREWORK_E2E_ES_MEMORY_MB`, `FIREWORK_E2E_KIBANA_VCPUS`, and
+  `FIREWORK_E2E_KIBANA_MEMORY_MB` when the host has different capacity.
+- `FIREWORK_E2E_HEALTH_RETRIES=80` controls the startup/restart threshold.
+  `FIREWORK_E2E_ES_JAVA_OPTS=-Xmx1g` is the compatibility default for the
+  currently published GitOps Elasticsearch image; a rebuilt image with the
+  current `fc-init` can use a normal multi-option value.
+- `FIREWORK_E2E_VOLUME_SIZE=2Gi` and
+  `FIREWORK_E2E_STORAGE_CAPACITY=8Gi` adjust the disposable local volume
+  pool.
+- `FIREWORK_E2E_ES_IMAGE_KEY` and `FIREWORK_E2E_KIBANA_IMAGE_KEY` select
+  alternate objects with the same GitOps rootfs contract.
+- `FIREWORK_E2E_FIRECRACKER_BIN` and `FIREWORK_E2E_KERNEL` optionally provide
+  local asset overrides; otherwise the pinned downloads are used.
+
+The runner creates a unique real S3 bucket for control-plane state and
+rendered node configs, records image/asset provenance, collects diagnostics
+before teardown, and deletes the bucket and Lima guest unless retention is
+requested. AWS credentials are passed through the process environment and are
+not written to the manifest or generated configuration files.

validation/e2e-local/scripts/check-env.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+die() {
+  printf 'ERROR: %s\n' "$*" >&2
+  exit 1
+}
+
+require_cmd() {
+  command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
+}
+
+require_cmd aws
+require_cmd curl
+require_cmd git
+require_cmd jq
+require_cmd openssl
+
+[[ -n "${AWS_REGION:-${AWS_DEFAULT_REGION:-}}" ]] || die "AWS_REGION or AWS_DEFAULT_REGION is required"
+
+case "$(uname -s)" in
+  Darwin)
+    require_cmd limactl
+    ;;
+  Linux)
+    require_cmd ip
+    require_cmd iptables
+    [[ -r /dev/kvm && -w /dev/kvm ]] || die "/dev/kvm is not readable and writable"
+    [[ -e /dev/net/tun ]] || die "/dev/net/tun is required"
+    ;;
+  *)
+    die "unsupported host OS: $(uname -s)"
+    ;;
+esac

validation/e2e-local/scripts/collect-diagnostics.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -u
+
+RUN_DIR=${1:?usage: collect-diagnostics.sh RUN_DIR}
+mkdir -p "$RUN_DIR/diagnostics"
+
+out="$RUN_DIR/diagnostics"
+
+for config in "$RUN_DIR"/config/*.yaml; do
+  [[ -f "$config" ]] || continue
+  name=$(basename "$config")
+  sed -E \
+    -e 's/^([[:space:]]*(operator_token|registry_bootstrap_token|github_webhook_secret):).*/\1 "<redacted>"/' \
+    -e 's/^([[:space:]]*-[[:space:]]*token:).*/\1 "<redacted>"/' \
+    "$config" > "$out/$name"
+done
+
+ip address show > "$out/ip-address.txt" 2>&1 || true
+ip route show table all > "$out/ip-routes.txt" 2>&1 || true
+iptables-save > "$out/iptables.txt" 2>&1 || true
+
+for namespace in fw-e2e-node-a fw-e2e-node-b; do
+  ip netns exec "$namespace" ip address show > "$out/$namespace-ip-address.txt" 2>&1 || true
+  ip netns exec "$namespace" ip route show table all > "$out/$namespace-ip-routes.txt" 2>&1 || true
+  ip netns exec "$namespace" iptables-save > "$out/$namespace-iptables.txt" 2>&1 || true
+done
+
+if [[ -n "${CONTROLPLANE_PID:-}" ]]; then
+  ps -o pid,ppid,state,etime,args -p "$CONTROLPLANE_PID" > "$out/controlplane-process.txt" 2>&1 || true
+fi
+for pid in ${AGENT_PIDS:-}; do
+  ps -o pid,ppid,state,etime,args -p "$pid" >> "$out/agent-processes.txt" 2>&1 || true
+done
+
+if [[ -n "${CONTROLPLANE_CURL_URL:-}" && -n "${CONTROLPLANE_CA_FILE:-}" ]]; then
+  curl --silent --show-error --cacert "$CONTROLPLANE_CA_FILE" \
+    -H "Authorization: Bearer ${CONTROLPLANE_OPERATOR_TOKEN:-}" \
+    "$CONTROLPLANE_CURL_URL/v1/nodes" > "$out/controlplane-nodes.json" 2>&1 || true
+  curl --silent --show-error --cacert "$CONTROLPLANE_CA_FILE" \
+    -H "Authorization: Bearer ${CONTROLPLANE_OPERATOR_TOKEN:-}" \
+    "$CONTROLPLANE_CURL_URL/v1/services" > "$out/controlplane-services.json" 2>&1 || true
+fi
+
+for endpoint in ${AGENT_ENDPOINTS:-}; do
+  name=${endpoint%%=*}
+  url=${endpoint#*=}
+  curl --silent --show-error --fail "$url/status" > "$out/${name}-status.json" 2>&1 || true
+done
+
+if [[ -n "${E2E_BUCKET:-}" ]]; then
+  aws s3api list-objects-v2 --bucket "$E2E_BUCKET" --prefix cp/v1/ \
+    --output json > "$out/s3-cp-v1-inventory.json" 2>&1 || true
+fi

validation/e2e-local/scripts/destroy-lab.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+manifest=${1:-}
+if [[ -z "$manifest" ]]; then
+  printf 'usage: make validate-e2e-local-clean FIREWORK_E2E_MANIFEST=/path/to/manifest.json\n' >&2
+  exit 2
+fi
+
+command -v jq >/dev/null 2>&1 || { printf 'ERROR: jq is required\n' >&2; exit 1; }
+command -v aws >/dev/null 2>&1 || { printf 'ERROR: aws is required\n' >&2; exit 1; }
+
+bucket=$(jq -r '.bucket // empty' "$manifest")
+region=$(jq -r '.region // empty' "$manifest")
+mode=$(jq -r '.mode // empty' "$manifest")
+instance=$(jq -r '.lima_instance // empty' "$manifest")
+
+[[ -n "$bucket" && -n "$region" ]] || { printf 'ERROR: invalid manifest: %s\n' "$manifest" >&2; exit 1; }
+export AWS_REGION="$region"
+export AWS_DEFAULT_REGION="$region"
+export AWS_EC2_METADATA_DISABLED=true
+
+if [[ "$mode" == lima && -n "$instance" ]] && command -v limactl >/dev/null 2>&1; then
+  limactl stop --force "$instance" >/dev/null 2>&1 || true
+  limactl delete --force "$instance" >/dev/null 2>&1 || true
+elif [[ "$mode" == linux ]]; then
+  for pid in $(jq -r '(.agent_pids[]?, .controlplane_pid?) | select(type == "number")' "$manifest"); do
+    sudo kill -TERM "$pid" >/dev/null 2>&1 || true
+  done
+  for namespace in fw-e2e-node-a fw-e2e-node-b; do
+    sudo ip netns del "$namespace" >/dev/null 2>&1 || true
+  done
+  sudo ip link del fw-e2e-br >/dev/null 2>&1 || true
+fi
+
+aws s3 rm "s3://$bucket" --recursive >/dev/null
+aws s3api delete-bucket --bucket "$bucket" >/dev/null
+printf 'destroyed local E2E lab from %s\n' "$manifest"

validation/e2e-local/scripts/generate-pki.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+OUT_DIR=${1:?usage: generate-pki.sh OUT_DIR}
+mkdir -p "$OUT_DIR"
+umask 077
+
+openssl req -x509 -newkey rsa:2048 -nodes \
+  -keyout "$OUT_DIR/ca.key" \
+  -out "$OUT_DIR/ca.crt" \
+  -days 2 \
+  -subj "/CN=firework-local-e2e-ca" \
+  >/dev/null 2>&1
+
+openssl req -newkey rsa:2048 -nodes \
+  -keyout "$OUT_DIR/controlplane.key" \
+  -out "$OUT_DIR/controlplane.csr" \
+  -subj "/CN=controlplane.local" \
+  >/dev/null 2>&1
+
+cat > "$OUT_DIR/controlplane.ext" <<'EOF'
+basicConstraints=CA:FALSE
+keyUsage=digitalSignature,keyEncipherment
+extendedKeyUsage=serverAuth
+subjectAltName=DNS:controlplane.local,IP:127.0.0.1,IP:10.254.0.1
+EOF
+
+openssl x509 -req \
+  -in "$OUT_DIR/controlplane.csr" \
+  -CA "$OUT_DIR/ca.crt" \
+  -CAkey "$OUT_DIR/ca.key" \
+  -CAcreateserial \
+  -out "$OUT_DIR/controlplane.crt" \
+  -days 2 \
+  -extfile "$OUT_DIR/controlplane.ext" \
+  >/dev/null 2>&1
+
+rm -f "$OUT_DIR/controlplane.csr" "$OUT_DIR/controlplane.ext" "$OUT_DIR/ca.srl"

validation/e2e-local/scripts/prepare-assets.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+RUN_DIR=${1:?usage: prepare-assets.sh RUN_DIR}
+IMAGE_DIR="$RUN_DIR/images"
+BIN_DIR="$RUN_DIR/bin"
+FIRECRACKER_VERSION="${FIREWORK_E2E_FIRECRACKER_VERSION:-1.12.0}"
+FIRECRACKER_ARCH="${FIREWORK_E2E_FIRECRACKER_ARCH:-aarch64}"
+FIRECRACKER_TARBALL="firecracker-v${FIRECRACKER_VERSION}-${FIRECRACKER_ARCH}.tgz"
+FIRECRACKER_SHA256="${FIREWORK_E2E_FIRECRACKER_SHA256:-55f3e76c6a16128e91aea1d2ed3d436f5d4e2e9547bfdd226ce570a89cd48921}"
+KERNEL_KEY="${FIREWORK_E2E_KERNEL_KEY:-firecracker-ci/v1.12/aarch64/vmlinux-5.10.233}"
+
+log() {
+  printf '==> %s\n' "$*"
+}
+
+die() {
+  printf 'ERROR: %s\n' "$*" >&2
+  exit 1
+}
+
+mkdir -p "$IMAGE_DIR" "$BIN_DIR"
+
+if [[ ! -x "$BIN_DIR/firecracker" ]]; then
+  tmp_dir="$(mktemp -d "$RUN_DIR/firecracker-download.XXXXXX")"
+  trap 'rm -rf "$tmp_dir"' EXIT
+  archive="$tmp_dir/$FIRECRACKER_TARBALL"
+  url="https://github.com/firecracker-microvm/firecracker/releases/download/v${FIRECRACKER_VERSION}/${FIRECRACKER_TARBALL}"
+  log "downloading pinned Firecracker $FIRECRACKER_VERSION ($FIRECRACKER_ARCH)"
+  curl --fail --silent --show-error --location "$url" --output "$archive"
+  printf '%s  %s\n' "$FIRECRACKER_SHA256" "$archive" | sha256sum --check --status - \
+    || die "Firecracker archive checksum mismatch: $url"
+  tar --extract --gzip --file "$archive" --directory "$tmp_dir"
+  extracted="$tmp_dir/release-v${FIRECRACKER_VERSION}-${FIRECRACKER_ARCH}/firecracker-v${FIRECRACKER_VERSION}-${FIRECRACKER_ARCH}"
+  [[ -x "$extracted" ]] || die "Firecracker archive did not contain $extracted"
+  install -m 0755 "$extracted" "$BIN_DIR/firecracker"
+fi
+
+[[ -x "$BIN_DIR/firecracker" ]] || die "Firecracker is not executable: $BIN_DIR/firecracker"
+
+kernel="$IMAGE_DIR/vmlinux"
+if [[ ! -r "$kernel" ]]; then
+  log "downloading pinned Firecracker kernel $KERNEL_KEY"
+  curl --fail --silent --show-error --location \
+    "https://s3.amazonaws.com/spec.ccfc.min/$KERNEL_KEY" --output "$kernel"
+  chmod 0644 "$kernel"
+fi
+
+[[ -s "$kernel" ]] || die "kernel is empty: $kernel"
+
+firecracker_version="$("$BIN_DIR/firecracker" --version 2>&1 | head -n 1)"
+kernel_sha256="$(sha256sum "$kernel" | awk '{print $1}')"
+firecracker_sha256="$(sha256sum "$BIN_DIR/firecracker" | awk '{print $1}')"
+cat > "$RUN_DIR/assets-manifest.json" <<EOF
+{
+  "firecracker_version": "$FIRECRACKER_VERSION",
+  "firecracker_arch": "$FIRECRACKER_ARCH",
+  "firecracker_release_sha256": "$FIRECRACKER_SHA256",
+  "firecracker_binary_sha256": "$firecracker_sha256",
+  "firecracker_reported_version": "$firecracker_version",
+  "kernel_key": "$KERNEL_KEY",
+  "kernel_sha256": "$kernel_sha256"
+}
+EOF
+
+log "prepared Firecracker and kernel assets"