diff --git a/Makefile b/Makefile index 7486b6c..78db0b2 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ LDFLAGS := -s -w \ -X '$(MODULE)/internal/version.Commit=$(COMMIT)' \ -X '$(MODULE)/internal/version.BuildTime=$(BUILD_TIME)' -.PHONY: all build-all build-agent build-controlplane build-configcheck build-fireworkctl build-fireworkctl-release build-fc-init build-linux-amd64 build-linux-arm64 clean test test-verbose test-race lint vet fmt tidy run smoke-local docker-build-controlplane-image docker-push-controlplane-image push-controlplane-image install help +.PHONY: all build-all build-agent build-controlplane build-configcheck build-fireworkctl build-fireworkctl-release build-fc-init build-linux-amd64 build-linux-arm64 clean test test-verbose test-race lint vet fmt tidy run smoke-local validate-e2e-local validate-e2e-local-clean docker-build-controlplane-image docker-push-controlplane-image push-controlplane-image install help all: build-all ## Alias for build-all @@ -118,6 +118,12 @@ run: build-agent ## Build and run with example config smoke-local: ## Run local smoke test with fake firecracker ./scripts/smoke-local.sh +validate-e2e-local: ## Run the local two-node E2E lab with real Firecracker and S3 + ./validation/e2e-local/scripts/validate.sh + +validate-e2e-local-clean: ## Destroy a retained local E2E lab (set FIREWORK_E2E_MANIFEST) + ./validation/e2e-local/scripts/destroy-lab.sh "$(FIREWORK_E2E_MANIFEST)" + docker-build-controlplane-image: ## Build control-plane image locally (linux/amd64) docker buildx build --platform linux/amd64 --file Dockerfile.controlplane \ --build-arg VERSION="$(VERSION)" \ diff --git a/internal/vm/manager.go b/internal/vm/manager.go index a0a32c7..b51ec11 100644 --- a/internal/vm/manager.go +++ b/internal/vm/manager.go @@ -538,6 +538,10 @@ func (m *Manager) writeVMConfig(vmDir string, svc config.ServiceConfig, prepared Drives: drives, MachineConfig: firecrackerMachineConfig{VCPUCount: svc.VCPUs, MemSizeMiB: svc.MemoryMB}, NetworkInterfaces: networkInterfaces, + // The VirtIO-RNG device prevents guests without a usable hardware + // random source from blocking application startup on /dev/random. + // This matters for arm64 guests nested inside Lima/VZ in particular. + Entropy: &firecrackerEntropyDevice{}, } configJSON, err := json.MarshalIndent(vmConfig, "", " ") if err != nil { @@ -556,6 +560,7 @@ type firecrackerConfig struct { Drives []firecrackerDrive `json:"drives"` MachineConfig firecrackerMachineConfig `json:"machine-config"` NetworkInterfaces []firecrackerNetworkInterface `json:"network-interfaces,omitempty"` + Entropy *firecrackerEntropyDevice `json:"entropy,omitempty"` } type firecrackerBootSource struct { @@ -575,6 +580,8 @@ type firecrackerMachineConfig struct { MemSizeMiB int `json:"mem_size_mib"` } +type firecrackerEntropyDevice struct{} + type firecrackerNetworkInterface struct { IfaceID string `json:"iface_id"` GuestMAC string `json:"guest_mac"` diff --git a/internal/vm/manager_test.go b/internal/vm/manager_test.go index b614b82..a6a6432 100644 --- a/internal/vm/manager_test.go +++ b/internal/vm/manager_test.go @@ -75,6 +75,9 @@ func TestWriteVMConfigAddsDeterministicVolumeDrivesAndPayload(t *testing.T) { if len(cfg.Drives) != 3 || cfg.Drives[1].PathOnHost != "/a.ext4" || cfg.Drives[2].PathOnHost != "/z.ext4" { t.Fatalf("unexpected drives: %#v", cfg.Drives) } + if cfg.Entropy == nil { + t.Fatal("expected a VirtIO-RNG entropy device") + } fields := strings.Fields(cfg.BootSource.BootArgs) var encoded string for i, field := range fields { diff --git a/validation/e2e-local/README.md b/validation/e2e-local/README.md new file mode 100644 index 0000000..addb10f --- /dev/null +++ b/validation/e2e-local/README.md @@ -0,0 +1,87 @@ +# Local stateful two-node E2E validation + +This harness runs one combined Firework setup inside an arm64 Lima Linux +guest on Apple Silicon: + +```text +one control plane (registry + events + controller + API) + | + real per-run S3 bucket + | + node-a namespace ---------------- node-b namespace + firework-agent firework-agent + Firecracker microVMs Firecracker microVMs +``` + +The workload uses the arm64 rootfs images produced by +`firework-gitops-example`: Elasticsearch and Kibana start together on node-a, +then the same desired state is changed to anti-affine placement and Kibana +moves to node-b. Both phases require eventual `running`/`healthy` convergence; +short startup or movement downtime is expected and allowed. Elasticsearch is +also driven to cluster status `green` after its single-node replica setting is +adjusted for this validation. + +The run additionally checks the real S3 state/rendered-config path, local +volume creation and reuse, agent restart/adoption, explicit empty desired +state, stale-node visibility and recovery, cross-node routing, and final +per-node port ownership. CI, Linux-native execution, and hosted-KVM probing +are Milestone 2 work and are not required by this local command today. + +## Prerequisites + +- Apple Silicon macOS with Lima 2.x and a Lima `vz` guest that exposes + readable/writable `/dev/kvm` and `/dev/net/tun`. +- Go and the repository build toolchain. +- AWS credentials able to create/list/write/delete a disposable S3 bucket and + read the workload-image bucket. +- Access to the existing arm64 GitOps image bucket. The default is + `artemnikitin-firework-images`; override it with + `FIREWORK_E2E_IMAGES_BUCKET` when needed. + +The harness downloads and verifies Firecracker 1.12.0 arm64 and uses the pinned +Firecracker CI kernel `firecracker-ci/v1.12/aarch64/vmlinux-5.10.233`. The VM +configuration enables Firecracker's VirtIO-RNG device so Java/Node workloads +do not wait indefinitely for guest entropy. Override either asset pin only for +an intentional compatibility investigation. The two workload rootfs images +are not copied into the repository or manually modified: the agents download +them through their production S3 image-sync path. + +## Run + +```bash +export AWS_REGION=us-east-1 +export FIREWORK_E2E_AWS_PROFILE=artemnikitin +make validate-e2e-local +``` + +Useful local options: + +- `FIREWORK_E2E_KEEP=1` retains the Lima guest, logs, manifest and disposable + S3 bucket for inspection. Clean it with + `make validate-e2e-local-clean FIREWORK_E2E_MANIFEST=`. +- `FIREWORK_E2E_TIMEOUT=1800` changes the bounded scenario timeout. The default + is intentionally generous because these production-sized rootfs images can + take several minutes to initialize under nested virtualization. +- `FIREWORK_E2E_LIMA_CPUS=8`, `FIREWORK_E2E_LIMA_MEMORY_GB=12`, and + `FIREWORK_E2E_LIMA_DISK_GB=60` size the local guest. +- By default Elasticsearch gets 4 vCPUs and 6 GiB, while Kibana gets 2 vCPUs + and 4 GiB. Override these with `FIREWORK_E2E_ES_VCPUS`, + `FIREWORK_E2E_ES_MEMORY_MB`, `FIREWORK_E2E_KIBANA_VCPUS`, and + `FIREWORK_E2E_KIBANA_MEMORY_MB` when the host has different capacity. +- `FIREWORK_E2E_HEALTH_RETRIES=80` controls the startup/restart threshold. + `FIREWORK_E2E_ES_JAVA_OPTS=-Xmx1g` is the compatibility default for the + currently published GitOps Elasticsearch image; a rebuilt image with the + current `fc-init` can use a normal multi-option value. +- `FIREWORK_E2E_VOLUME_SIZE=2Gi` and + `FIREWORK_E2E_STORAGE_CAPACITY=8Gi` adjust the disposable local volume + pool. +- `FIREWORK_E2E_ES_IMAGE_KEY` and `FIREWORK_E2E_KIBANA_IMAGE_KEY` select + alternate objects with the same GitOps rootfs contract. +- `FIREWORK_E2E_FIRECRACKER_BIN` and `FIREWORK_E2E_KERNEL` optionally provide + local asset overrides; otherwise the pinned downloads are used. + +The runner creates a unique real S3 bucket for control-plane state and +rendered node configs, records image/asset provenance, collects diagnostics +before teardown, and deletes the bucket and Lima guest unless retention is +requested. AWS credentials are passed through the process environment and are +not written to the manifest or generated configuration files. diff --git a/validation/e2e-local/scripts/check-env.sh b/validation/e2e-local/scripts/check-env.sh new file mode 100755 index 0000000..51b149f --- /dev/null +++ b/validation/e2e-local/scripts/check-env.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "missing required command: $1" +} + +require_cmd aws +require_cmd curl +require_cmd git +require_cmd jq +require_cmd openssl + +[[ -n "${AWS_REGION:-${AWS_DEFAULT_REGION:-}}" ]] || die "AWS_REGION or AWS_DEFAULT_REGION is required" + +case "$(uname -s)" in + Darwin) + require_cmd limactl + ;; + Linux) + require_cmd ip + require_cmd iptables + [[ -r /dev/kvm && -w /dev/kvm ]] || die "/dev/kvm is not readable and writable" + [[ -e /dev/net/tun ]] || die "/dev/net/tun is required" + ;; + *) + die "unsupported host OS: $(uname -s)" + ;; +esac diff --git a/validation/e2e-local/scripts/collect-diagnostics.sh b/validation/e2e-local/scripts/collect-diagnostics.sh new file mode 100755 index 0000000..5e8922d --- /dev/null +++ b/validation/e2e-local/scripts/collect-diagnostics.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -u + +RUN_DIR=${1:?usage: collect-diagnostics.sh RUN_DIR} +mkdir -p "$RUN_DIR/diagnostics" + +out="$RUN_DIR/diagnostics" + +for config in "$RUN_DIR"/config/*.yaml; do + [[ -f "$config" ]] || continue + name=$(basename "$config") + sed -E \ + -e 's/^([[:space:]]*(operator_token|registry_bootstrap_token|github_webhook_secret):).*/\1 ""/' \ + -e 's/^([[:space:]]*-[[:space:]]*token:).*/\1 ""/' \ + "$config" > "$out/$name" +done + +ip address show > "$out/ip-address.txt" 2>&1 || true +ip route show table all > "$out/ip-routes.txt" 2>&1 || true +iptables-save > "$out/iptables.txt" 2>&1 || true + +for namespace in fw-e2e-node-a fw-e2e-node-b; do + ip netns exec "$namespace" ip address show > "$out/$namespace-ip-address.txt" 2>&1 || true + ip netns exec "$namespace" ip route show table all > "$out/$namespace-ip-routes.txt" 2>&1 || true + ip netns exec "$namespace" iptables-save > "$out/$namespace-iptables.txt" 2>&1 || true +done + +if [[ -n "${CONTROLPLANE_PID:-}" ]]; then + ps -o pid,ppid,state,etime,args -p "$CONTROLPLANE_PID" > "$out/controlplane-process.txt" 2>&1 || true +fi +for pid in ${AGENT_PIDS:-}; do + ps -o pid,ppid,state,etime,args -p "$pid" >> "$out/agent-processes.txt" 2>&1 || true +done + +if [[ -n "${CONTROLPLANE_CURL_URL:-}" && -n "${CONTROLPLANE_CA_FILE:-}" ]]; then + curl --silent --show-error --cacert "$CONTROLPLANE_CA_FILE" \ + -H "Authorization: Bearer ${CONTROLPLANE_OPERATOR_TOKEN:-}" \ + "$CONTROLPLANE_CURL_URL/v1/nodes" > "$out/controlplane-nodes.json" 2>&1 || true + curl --silent --show-error --cacert "$CONTROLPLANE_CA_FILE" \ + -H "Authorization: Bearer ${CONTROLPLANE_OPERATOR_TOKEN:-}" \ + "$CONTROLPLANE_CURL_URL/v1/services" > "$out/controlplane-services.json" 2>&1 || true +fi + +for endpoint in ${AGENT_ENDPOINTS:-}; do + name=${endpoint%%=*} + url=${endpoint#*=} + curl --silent --show-error --fail "$url/status" > "$out/${name}-status.json" 2>&1 || true +done + +if [[ -n "${E2E_BUCKET:-}" ]]; then + aws s3api list-objects-v2 --bucket "$E2E_BUCKET" --prefix cp/v1/ \ + --output json > "$out/s3-cp-v1-inventory.json" 2>&1 || true +fi diff --git a/validation/e2e-local/scripts/destroy-lab.sh b/validation/e2e-local/scripts/destroy-lab.sh new file mode 100755 index 0000000..60b30ef --- /dev/null +++ b/validation/e2e-local/scripts/destroy-lab.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +manifest=${1:-} +if [[ -z "$manifest" ]]; then + printf 'usage: make validate-e2e-local-clean FIREWORK_E2E_MANIFEST=/path/to/manifest.json\n' >&2 + exit 2 +fi + +command -v jq >/dev/null 2>&1 || { printf 'ERROR: jq is required\n' >&2; exit 1; } +command -v aws >/dev/null 2>&1 || { printf 'ERROR: aws is required\n' >&2; exit 1; } + +bucket=$(jq -r '.bucket // empty' "$manifest") +region=$(jq -r '.region // empty' "$manifest") +mode=$(jq -r '.mode // empty' "$manifest") +instance=$(jq -r '.lima_instance // empty' "$manifest") + +[[ -n "$bucket" && -n "$region" ]] || { printf 'ERROR: invalid manifest: %s\n' "$manifest" >&2; exit 1; } +export AWS_REGION="$region" +export AWS_DEFAULT_REGION="$region" +export AWS_EC2_METADATA_DISABLED=true + +if [[ "$mode" == lima && -n "$instance" ]] && command -v limactl >/dev/null 2>&1; then + limactl stop --force "$instance" >/dev/null 2>&1 || true + limactl delete --force "$instance" >/dev/null 2>&1 || true +elif [[ "$mode" == linux ]]; then + for pid in $(jq -r '(.agent_pids[]?, .controlplane_pid?) | select(type == "number")' "$manifest"); do + sudo kill -TERM "$pid" >/dev/null 2>&1 || true + done + for namespace in fw-e2e-node-a fw-e2e-node-b; do + sudo ip netns del "$namespace" >/dev/null 2>&1 || true + done + sudo ip link del fw-e2e-br >/dev/null 2>&1 || true +fi + +aws s3 rm "s3://$bucket" --recursive >/dev/null +aws s3api delete-bucket --bucket "$bucket" >/dev/null +printf 'destroyed local E2E lab from %s\n' "$manifest" diff --git a/validation/e2e-local/scripts/generate-pki.sh b/validation/e2e-local/scripts/generate-pki.sh new file mode 100755 index 0000000..142e3f3 --- /dev/null +++ b/validation/e2e-local/scripts/generate-pki.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +OUT_DIR=${1:?usage: generate-pki.sh OUT_DIR} +mkdir -p "$OUT_DIR" +umask 077 + +openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$OUT_DIR/ca.key" \ + -out "$OUT_DIR/ca.crt" \ + -days 2 \ + -subj "/CN=firework-local-e2e-ca" \ + >/dev/null 2>&1 + +openssl req -newkey rsa:2048 -nodes \ + -keyout "$OUT_DIR/controlplane.key" \ + -out "$OUT_DIR/controlplane.csr" \ + -subj "/CN=controlplane.local" \ + >/dev/null 2>&1 + +cat > "$OUT_DIR/controlplane.ext" <<'EOF' +basicConstraints=CA:FALSE +keyUsage=digitalSignature,keyEncipherment +extendedKeyUsage=serverAuth +subjectAltName=DNS:controlplane.local,IP:127.0.0.1,IP:10.254.0.1 +EOF + +openssl x509 -req \ + -in "$OUT_DIR/controlplane.csr" \ + -CA "$OUT_DIR/ca.crt" \ + -CAkey "$OUT_DIR/ca.key" \ + -CAcreateserial \ + -out "$OUT_DIR/controlplane.crt" \ + -days 2 \ + -extfile "$OUT_DIR/controlplane.ext" \ + >/dev/null 2>&1 + +rm -f "$OUT_DIR/controlplane.csr" "$OUT_DIR/controlplane.ext" "$OUT_DIR/ca.srl" diff --git a/validation/e2e-local/scripts/prepare-assets.sh b/validation/e2e-local/scripts/prepare-assets.sh new file mode 100755 index 0000000..ea26058 --- /dev/null +++ b/validation/e2e-local/scripts/prepare-assets.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +RUN_DIR=${1:?usage: prepare-assets.sh RUN_DIR} +IMAGE_DIR="$RUN_DIR/images" +BIN_DIR="$RUN_DIR/bin" +FIRECRACKER_VERSION="${FIREWORK_E2E_FIRECRACKER_VERSION:-1.12.0}" +FIRECRACKER_ARCH="${FIREWORK_E2E_FIRECRACKER_ARCH:-aarch64}" +FIRECRACKER_TARBALL="firecracker-v${FIRECRACKER_VERSION}-${FIRECRACKER_ARCH}.tgz" +FIRECRACKER_SHA256="${FIREWORK_E2E_FIRECRACKER_SHA256:-55f3e76c6a16128e91aea1d2ed3d436f5d4e2e9547bfdd226ce570a89cd48921}" +KERNEL_KEY="${FIREWORK_E2E_KERNEL_KEY:-firecracker-ci/v1.12/aarch64/vmlinux-5.10.233}" + +log() { + printf '==> %s\n' "$*" +} + +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +mkdir -p "$IMAGE_DIR" "$BIN_DIR" + +if [[ ! -x "$BIN_DIR/firecracker" ]]; then + tmp_dir="$(mktemp -d "$RUN_DIR/firecracker-download.XXXXXX")" + trap 'rm -rf "$tmp_dir"' EXIT + archive="$tmp_dir/$FIRECRACKER_TARBALL" + url="https://github.com/firecracker-microvm/firecracker/releases/download/v${FIRECRACKER_VERSION}/${FIRECRACKER_TARBALL}" + log "downloading pinned Firecracker $FIRECRACKER_VERSION ($FIRECRACKER_ARCH)" + curl --fail --silent --show-error --location "$url" --output "$archive" + printf '%s %s\n' "$FIRECRACKER_SHA256" "$archive" | sha256sum --check --status - \ + || die "Firecracker archive checksum mismatch: $url" + tar --extract --gzip --file "$archive" --directory "$tmp_dir" + extracted="$tmp_dir/release-v${FIRECRACKER_VERSION}-${FIRECRACKER_ARCH}/firecracker-v${FIRECRACKER_VERSION}-${FIRECRACKER_ARCH}" + [[ -x "$extracted" ]] || die "Firecracker archive did not contain $extracted" + install -m 0755 "$extracted" "$BIN_DIR/firecracker" +fi + +[[ -x "$BIN_DIR/firecracker" ]] || die "Firecracker is not executable: $BIN_DIR/firecracker" + +kernel="$IMAGE_DIR/vmlinux" +if [[ ! -r "$kernel" ]]; then + log "downloading pinned Firecracker kernel $KERNEL_KEY" + curl --fail --silent --show-error --location \ + "https://s3.amazonaws.com/spec.ccfc.min/$KERNEL_KEY" --output "$kernel" + chmod 0644 "$kernel" +fi + +[[ -s "$kernel" ]] || die "kernel is empty: $kernel" + +firecracker_version="$("$BIN_DIR/firecracker" --version 2>&1 | head -n 1)" +kernel_sha256="$(sha256sum "$kernel" | awk '{print $1}')" +firecracker_sha256="$(sha256sum "$BIN_DIR/firecracker" | awk '{print $1}')" +cat > "$RUN_DIR/assets-manifest.json" < %s\n' "$*" +} + +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +case "$(uname -m)" in + x86_64) BIN_ARCH=amd64 ;; + aarch64|arm64) BIN_ARCH=arm64 ;; + *) die "unsupported Linux architecture in lab: $(uname -m)" ;; +esac + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "missing required command in lab: $1" +} + +install_guest_tools() { + local missing=0 + for command_name in aws curl git ip iptables jq openssl sha256sum tar; do + if ! command -v "$command_name" >/dev/null 2>&1; then + missing=1 + fi + done + if (( missing == 0 )); then + return + fi + command -v apt-get >/dev/null 2>&1 || die "lab tools are missing and apt-get is unavailable" + log "installing missing Linux lab tools" + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq awscli ca-certificates curl e2fsprogs file git iproute2 iptables jq openssl tar +} + +prepare_direct_launcher_path() { + local command_name command_path + mkdir -p "$DIRECT_BIN_DIR" + # Lima's Ubuntu guest has systemd, so the agent would otherwise select + # systemd-run. Its transient-unit PID exposes systemd-executor through + # /proc//exe, which cannot satisfy Firecracker ownership checks. Keep + # the agent's PATH deliberately small so chooseLauncher selects the direct + # process launcher while retaining the host tools used by networking and + # local-volume management. + for command_name in \ + e2fsck ip iptables mount mountpoint mkfs.ext4 resize2fs sh sysctl \ + tune2fs umount; do + command_path="$(command -v "$command_name" || true)" + [[ -n "$command_path" ]] || die "missing direct-launcher dependency: $command_name" + ln -sf "$command_path" "$DIRECT_BIN_DIR/$command_name" + done +} + +write_file() { + local path="$1" + mkdir -p "$(dirname "$path")" + cat > "$path" +} + +cleanup_rootfs_mount() { + if mountpoint -q "$ROOTFS_MOUNT"; then + sync || true + umount "$ROOTFS_MOUNT" || true + fi +} + +prepare_assets() { + [[ -x "$ASSET_SCRIPT" ]] || die "asset preparation script is missing: $ASSET_SCRIPT" + FIREWORK_E2E_FIRECRACKER_BIN="$FIRECRACKER" FIREWORK_E2E_KERNEL="$KERNEL" \ + FIREWORK_E2E_FIRECRACKER_VERSION="${FIREWORK_E2E_FIRECRACKER_VERSION:-1.12.0}" \ + FIREWORK_E2E_KERNEL_KEY="${FIREWORK_E2E_KERNEL_KEY:-firecracker-ci/v1.12/aarch64/vmlinux-5.10.233}" \ + bash "$ASSET_SCRIPT" "$RUN_DIR" + [[ -x "$FIRECRACKER" ]] || die "prepared Firecracker binary is missing" + [[ -r "$KERNEL" ]] || die "prepared kernel is missing" +} + +setup_git_repo() { + mkdir -p "$CONFIG_DIR/repo/services" + write_file "$CONFIG_DIR/repo/defaults.yaml" </dev/null 2>&1 || { + git -C "$CONFIG_DIR/repo" init >/dev/null + git -C "$CONFIG_DIR/repo" checkout -b main >/dev/null + } + git -C "$CONFIG_DIR/repo" config user.name firework-local-e2e + git -C "$CONFIG_DIR/repo" config user.email firework-local-e2e@localhost + git -C "$CONFIG_DIR/repo" add . + git -C "$CONFIG_DIR/repo" commit -m "local e2e stateful workload" >/dev/null +} + +write_stateful_services() { + local anti_affinity="${1:-}" + local volume="" + if [[ "${FIREWORK_E2E_ENABLE_VOLUME:-1}" == 1 ]]; then + volume="$(printf 'volumes:\n - name: data\n type: local\n mount_path: /usr/share/elasticsearch/data\n size: %s\n' "$VOLUME_SIZE")" + fi + write_file "$CONFIG_DIR/repo/services/elasticsearch.yaml" </dev/null +} + +setup_network() { + local namespace ip_address uplink resolver_source + resolver_source="/run/systemd/resolve/resolv.conf" + if [[ ! -r "$resolver_source" ]]; then + resolver_source="$RUN_DIR/resolv.conf" + cp /etc/resolv.conf "$resolver_source" + sed -i 's/127\.0\.0\.53/192.168.5.2/g' "$resolver_source" + fi + ip link add name fw-e2e-br type bridge + ip addr add 10.254.0.1/24 dev fw-e2e-br + ip link set fw-e2e-br up + sysctl -w net.ipv4.ip_forward=1 >/dev/null + for node in a b; do + namespace="fw-e2e-node-${node}" + if [[ "$node" == a ]]; then ip_address=10.254.0.11; else ip_address=10.254.0.12; fi + uplink="fw-e2e-${node}-up" + ip netns add "$namespace" + ip link add "fw-e2e-${node}" type veth peer name "$uplink" + ip link set "fw-e2e-${node}" master fw-e2e-br + ip link set "fw-e2e-${node}" up + ip link set "$uplink" netns "$namespace" + ip -n "$namespace" link set lo up + ip -n "$namespace" link set "$uplink" up + ip -n "$namespace" addr add "$ip_address/24" dev "$uplink" + ip -n "$namespace" route add default via 10.254.0.1 + mkdir -p "/etc/netns/$namespace" + cp "$resolver_source" "/etc/netns/$namespace/resolv.conf" + ip netns exec "$namespace" sysctl -w net.ipv4.ip_forward=1 >/dev/null + ip netns exec "$namespace" sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null + ip netns exec "$namespace" sysctl -w net.ipv4.conf.default.rp_filter=0 >/dev/null + done + mkdir -p "$RUN_DIR/storage/node-a" "$RUN_DIR/storage/node-b" + storage_mount_size="${STORAGE_CAPACITY//i/}" + mount -t tmpfs -o "size=$storage_mount_size" firework-e2e-storage-a "$RUN_DIR/storage/node-a" + mount -t tmpfs -o "size=$storage_mount_size" firework-e2e-storage-b "$RUN_DIR/storage/node-b" + ROOT_OUT_INTERFACE="$(ip -o route show default | awk 'NR == 1 {print $5}')" + [[ -n "$ROOT_OUT_INTERFACE" ]] || die "could not determine the Lima guest default interface" + iptables -t nat -A POSTROUTING -s 10.254.0.0/24 -o "$ROOT_OUT_INTERFACE" -j MASQUERADE + iptables -A FORWARD -s 10.254.0.0/24 -j ACCEPT + iptables -A FORWARD -d 10.254.0.0/24 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT + ip route add 172.16.1.0/24 via 10.254.0.11 dev fw-e2e-br + ip route add 172.16.2.0/24 via 10.254.0.12 dev fw-e2e-br +} + +write_controlplane_config() { + local operator_token="$1" + write_file "$CONFIG_DIR/controlplane.yaml" </dev/null 2>&1; then return 0; fi + elif [[ -n "$ca_file" ]]; then + if curl --silent --show-error --fail --cacert "$ca_file" "$url" >/dev/null 2>&1; then return 0; fi + elif curl --silent --show-error --fail "$url" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +api_get() { + local path="$1" + curl --silent --show-error --fail --cacert "$PKI_DIR/ca.crt" \ + -H "Authorization: Bearer $OPERATOR_TOKEN" "https://127.0.0.1:9445$path" +} + +wait_nodes() { + local count="$1" deadline=$((SECONDS + E2E_TIMEOUT)) nodes + while (( SECONDS < deadline )); do + nodes="$(api_get /v1/nodes 2>/dev/null || true)" + if jq -e --argjson count "$count" '.count == $count and all(.items[]; .state == "ready")' <<<"$nodes" >/dev/null 2>&1; then + printf '%s\n' "$nodes" + return 0 + fi + sleep 2 + done + printf '%s\n' "${nodes:-{}}" > "$RUN_DIR/nodes-timeout.json" + return 1 +} + +wait_services_healthy() { + local expected="$1" distinct_nodes="$2" deadline=$((SECONDS + E2E_TIMEOUT)) services + while (( SECONDS < deadline )); do + services="$(api_get /v1/services 2>/dev/null || true)" + if jq -e --argjson expected "$expected" --argjson distinct "$distinct_nodes" \ + '.count == $expected and all(.items[]; .state == "running" and .health == "healthy") and (([.items[].node] | unique | length) == $distinct)' \ + <<<"$services" >/dev/null 2>&1; then + printf '%s\n' "$services" + return 0 + fi + sleep 2 + done + printf '%s\n' "${services:-{}}" > "$RUN_DIR/services-timeout.json" + return 1 +} + +wait_empty_state() { + local deadline=$((SECONDS + E2E_TIMEOUT)) services status + while (( SECONDS < deadline )); do + services="$(api_get /v1/services 2>/dev/null || true)" + status="$(curl --silent --show-error --fail http://10.254.0.11:18081/status 2>/dev/null || true)" + if jq -e '.count == 0' <<<"$services" >/dev/null 2>&1 && \ + jq -e '.desired_services == 0 and (.services | length == 0)' <<<"$status" >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + return 1 +} + +service_node() { + local name="$1" services + services="$(api_get /v1/services)" + jq -r --arg name "$name" '.items[] | select(.name == $name) | .node' <<<"$services" +} + +node_host() { + case "$1" in + node-a) printf '10.254.0.11\n' ;; + node-b) printf '10.254.0.12\n' ;; + *) return 1 ;; + esac +} + +service_url() { + local name="$1" port="$2" node + node="$(service_node "$name")" + printf 'http://%s:%s\n' "$(node_host "$node")" "$port" +} + +wait_service_http() { + local url="$1" deadline=$((SECONDS + E2E_TIMEOUT)) + while (( SECONDS < deadline )); do + if curl --silent --show-error --fail "$url" >/dev/null 2>&1; then return 0; fi + sleep 2 + done + return 1 +} + +assert_elasticsearch_green() { + local endpoint="$1" response + wait_service_http "$endpoint/_cluster/health" || die "Elasticsearch endpoint did not become reachable: $endpoint" + curl --silent --show-error --fail -X PUT "$endpoint/_settings" \ + -H 'Content-Type: application/json' \ + --data '{"index":{"number_of_replicas":0}}' >/dev/null + local deadline=$((SECONDS + E2E_TIMEOUT)) + while (( SECONDS < deadline )); do + response="$(curl --silent --show-error --fail "$endpoint/_cluster/health" 2>/dev/null || true)" + if jq -e '.status == "green"' <<<"$response" >/dev/null 2>&1; then + printf '%s\n' "$response" > "$RUN_DIR/elasticsearch-health.json" + return 0 + fi + sleep 5 + done + printf '%s\n' "${response:-{}}" > "$RUN_DIR/elasticsearch-health-timeout.json" + return 1 +} + +send_webhook() { + local delivery_id="$1" payload signature + payload="$(jq -cn --arg ref refs/heads/main --arg url "file://$CONFIG_DIR/repo" \ + '{ref:$ref,repository:{clone_url:$url}}')" + signature="$(printf '%s' "$payload" | openssl dgst -sha256 -hmac "$WEBHOOK_SECRET" -binary | od -An -vtx1 | tr -d ' \n')" + curl --silent --show-error --fail --cacert "$PKI_DIR/ca.crt" \ + --resolve controlplane.local:9444:10.254.0.1 \ + -H 'Content-Type: application/json' \ + -H 'X-GitHub-Event: push' \ + -H "X-GitHub-Delivery: $delivery_id" \ + -H "X-Hub-Signature-256: sha256=$signature" \ + --data "$payload" https://controlplane.local:9444/v1/events/github >/dev/null +} + +start_agent() { + local node="$1" + PATH="$DIRECT_BIN_DIR" ip netns exec "fw-e2e-$node" "$BIN_DIR/firework-agent-linux-$BIN_ARCH" \ + --config "$CONFIG_DIR/agent-$node.yaml" > "$LOG_DIR/$node.log" 2>&1 & + AGENT_PIDS+=("$!") +} + +stop_agent() { + local index="$1" + local pid="${AGENT_PIDS[$index]:-}" + [[ -n "$pid" ]] || return 0 + kill -TERM "$pid" >/dev/null 2>&1 || true + wait "$pid" >/dev/null 2>&1 || true + AGENT_PIDS[index]="" +} + +commit_and_notify() { + local message="$1" delivery="$2" + commit_stateful_update "$message" + send_webhook "$delivery" +} + +stop_processes() { + local pid + for pid in "${AGENT_PIDS[@]}" "$CONTROLPLANE_PID"; do + [[ -n "$pid" ]] || continue + kill -TERM "$pid" >/dev/null 2>&1 || true + done + for pid in "${AGENT_PIDS[@]}" "$CONTROLPLANE_PID"; do + [[ -n "$pid" ]] || continue + wait "$pid" >/dev/null 2>&1 || true + done + AGENT_PIDS=() + CONTROLPLANE_PID="" +} + +delete_network() { + ip route del 172.16.1.0/24 via 10.254.0.11 dev fw-e2e-br >/dev/null 2>&1 || true + ip route del 172.16.2.0/24 via 10.254.0.12 dev fw-e2e-br >/dev/null 2>&1 || true + ip netns del fw-e2e-node-a >/dev/null 2>&1 || true + ip netns del fw-e2e-node-b >/dev/null 2>&1 || true + rm -rf /etc/netns/fw-e2e-node-a /etc/netns/fw-e2e-node-b + umount "$RUN_DIR/storage/node-a" >/dev/null 2>&1 || true + umount "$RUN_DIR/storage/node-b" >/dev/null 2>&1 || true + if [[ -n "$ROOT_OUT_INTERFACE" ]]; then + iptables -t nat -D POSTROUTING -s 10.254.0.0/24 -o "$ROOT_OUT_INTERFACE" -j MASQUERADE >/dev/null 2>&1 || true + iptables -D FORWARD -s 10.254.0.0/24 -j ACCEPT >/dev/null 2>&1 || true + iptables -D FORWARD -d 10.254.0.0/24 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT >/dev/null 2>&1 || true + fi + ip link del fw-e2e-br >/dev/null 2>&1 || true +} + +write_manifest() { + local status="$1" + local agent_pids_json + agent_pids_json="$(printf '%s\n' "${AGENT_PIDS[@]}" | jq -Rsc 'split("\n") | map(select(length > 0) | tonumber)')" + jq -n \ + --arg status "$status" \ + --arg mode "$RUN_MODE" \ + --arg commit "$FIREWORK_COMMIT" \ + --arg bucket "$E2E_BUCKET" \ + --arg images_bucket "$IMAGES_BUCKET" \ + --arg es_image "$ES_IMAGE_KEY" \ + --arg kibana_image "$KIBANA_IMAGE_KEY" \ + --arg region "$AWS_REGION_VALUE" \ + --arg run_dir "$RUN_DIR" \ + --arg controlplane_pid "${CONTROLPLANE_PID:-}" \ + --argjson agent_pids "$agent_pids_json" \ + '{status:$status,mode:$mode,scenario:"stateful",firework_commit:$commit,bucket:$bucket,images_bucket:$images_bucket,images:{elasticsearch:$es_image,kibana:$kibana_image},region:$region,run_dir:$run_dir,controlplane_pid:$controlplane_pid,agent_pids:$agent_pids}' \ + > "$RUN_DIR/manifest.json" +} + +cleanup() { + local status=$? + set +e + export CONTROLPLANE_CURL_URL="https://127.0.0.1:9445" + export CONTROLPLANE_CA_FILE="$PKI_DIR/ca.crt" + export CONTROLPLANE_OPERATOR_TOKEN="${OPERATOR_TOKEN:-}" + export E2E_BUCKET + export AGENT_ENDPOINTS="node-a=http://10.254.0.11:18081 node-b=http://10.254.0.12:18082" + "$SCRIPT_DIR/collect-diagnostics.sh" "$RUN_DIR" || true + if [[ "$status" -eq 0 ]]; then SCENARIO_STATUS="passed"; fi + cleanup_rootfs_mount + write_manifest "$SCENARIO_STATUS" + if [[ "$KEEP_LAB" != "1" ]]; then + stop_processes + delete_network + else + log "retaining local E2E lab at $RUN_DIR" + log "cleanup with: make validate-e2e-local-clean FIREWORK_E2E_MANIFEST=$RUN_DIR/manifest.json" + fi + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT TERM + +install_guest_tools +for command_name in mountpoint mount umount install; do require_cmd "$command_name"; done +[[ "$(id -u)" -eq 0 ]] || die "the Linux lab runner must execute as root" +[[ -r /dev/kvm && -w /dev/kvm ]] || die "/dev/kvm is not readable and writable in the lab" +[[ -e /dev/net/tun ]] || die "/dev/net/tun is required in the lab" + +log "preparing Firecracker assets" +prepare_assets +"$SCRIPT_DIR/generate-pki.sh" "$PKI_DIR" +setup_git_repo +setup_network +prepare_direct_launcher_path + +OPERATOR_TOKEN="local-e2e-operator-$FIREWORK_COMMIT" +write_controlplane_config "$OPERATOR_TOKEN" +write_agent_config node-a 10.254.0.11 172.16.1.0/24 172.16.1.1 "local-e2e-node-a-$FIREWORK_COMMIT" +write_agent_config node-b 10.254.0.12 172.16.2.0/24 172.16.2.1 "local-e2e-node-b-$FIREWORK_COMMIT" + +log "starting combined control plane" +"$BIN_DIR/firework-controlplane-linux-$BIN_ARCH" --config "$CONFIG_DIR/controlplane.yaml" > "$LOG_DIR/controlplane.log" 2>&1 & +CONTROLPLANE_PID=$! +wait_http "https://127.0.0.1:9445/healthz" "$PKI_DIR/ca.crt" || die "control-plane API did not become healthy" + +log "starting node-a and proving the colocated stateful topology" +start_agent node-a +wait_http "http://10.254.0.11:18081/healthz" "" || die "node-a API did not become healthy" +nodes_json="$(wait_nodes 1)" || die "node-a did not become ready" +services_json="$(wait_services_healthy 2 1)" || die "stateful services did not converge in the colocated topology" +printf '%s\n' "$nodes_json" > "$RUN_DIR/nodes-colocated.json" +printf '%s\n' "$services_json" > "$RUN_DIR/services-colocated.json" +es_endpoint="$(service_url tenant-2-elasticsearch 19200)" +kibana_endpoint="$(service_url tenant-2-kibana 15612)" +assert_elasticsearch_green "$es_endpoint" || die "Elasticsearch did not become green in the colocated topology" +wait_service_http "$kibana_endpoint/api/status" || die "Kibana did not become reachable in the colocated topology" +[[ -s "$IMAGE_DIR/$ES_IMAGE_KEY" && -s "$IMAGE_DIR/$KIBANA_IMAGE_KEY" ]] \ + || die "the agent did not download both real S3 workload images" + +log "starting node-b and checking that the colocated assignment remains stable" +start_agent node-b +wait_http "http://10.254.0.12:18082/healthz" "" || die "node-b API did not become healthy" +nodes_json="$(wait_nodes 2)" || die "two nodes did not become ready" +services_json="$(wait_services_healthy 2 1)" || die "services unexpectedly failed while node-b enrolled" +printf '%s\n' "$nodes_json" > "$RUN_DIR/nodes-two-ready.json" +printf '%s\n' "$services_json" > "$RUN_DIR/services-colocated-two-nodes.json" + +log "updating the same stateful workload with anti-affinity and proving movement" +write_stateful_services 'anti_affinity_group: "stateful"' +commit_and_notify "split stateful services across nodes" "split-$FIREWORK_COMMIT" +services_json="$(wait_services_healthy 2 2)" || die "stateful services did not converge after movement" +printf '%s\n' "$services_json" > "$RUN_DIR/services-split.json" +es_endpoint="$(service_url tenant-2-elasticsearch 19200)" +kibana_endpoint="$(service_url tenant-2-kibana 15612)" +assert_elasticsearch_green "$es_endpoint" || die "Elasticsearch did not become green after movement" +wait_service_http "$kibana_endpoint/api/status" || die "Kibana did not become reachable after movement" + +log "restarting node-a and verifying surviving VM and local-volume adoption" +marker="local-validation-$(date -u +%Y%m%dT%H%M%SZ)" +curl --silent --show-error --fail -X PUT "$es_endpoint/local-validation/_doc/marker" \ + -H 'Content-Type: application/json' --data "{\"marker\":\"$marker\"}" >/dev/null +stop_agent 0 +start_agent node-a +wait_http "http://10.254.0.11:18081/healthz" "" || die "node-a did not recover after agent restart" +services_json="$(wait_services_healthy 2 2)" || die "services did not converge after agent restart" +printf '%s\n' "$services_json" > "$RUN_DIR/services-after-agent-restart.json" +curl --silent --show-error --fail "$es_endpoint/local-validation/_doc/marker" \ + | jq -e --arg marker "$marker" '._source.marker == $marker' >/dev/null 2>&1 \ + || die "Elasticsearch volume marker was not readable after agent restart" + +log "removing the desired state and checking explicit empty assignment convergence" +rm -f "$CONFIG_DIR/repo/services"/*.yaml +commit_and_notify "remove stateful workload" "empty-$FIREWORK_COMMIT" +wait_empty_state || die "agents did not converge to an explicit empty desired state" + +log "restoring the stateful workload for stale-node and port ownership checks" +write_stateful_services 'anti_affinity_group: "stateful"' +commit_and_notify "restore stateful workload" "restore-$FIREWORK_COMMIT" +services_json="$(wait_services_healthy 2 2)" || die "stateful workload did not restore after empty state" + +log "stopping node-b and asserting stale-node visibility before recovery" +stop_agent 1 +stale_deadline=$((SECONDS + 90)) +stale_seen=0 +while (( SECONDS < stale_deadline )); do + nodes_json="$(api_get /v1/nodes 2>/dev/null || true)" + if jq -e '.items[] | select(.node_id == "node-b" and .state == "down")' <<<"$nodes_json" >/dev/null 2>&1; then + stale_seen=1 + break + fi + sleep 2 +done +[[ "$stale_seen" == 1 ]] || die "node-b did not become visibly down after heartbeat loss" +start_agent node-b +wait_http "http://10.254.0.12:18082/healthz" "" || die "node-b did not recover after stale-node check" +services_json="$(wait_services_healthy 2 2)" || die "services did not reconverge after stale-node recovery" + +log "checking per-node host-port ownership and rendered S3 state" +es_node="$(service_node tenant-2-elasticsearch)" +kibana_node="$(service_node tenant-2-kibana)" +[[ "$es_node" != "$kibana_node" ]] || die "anti-affinity placement regressed: both services are on $es_node" +jq -e '[.items[].port_forwards[]?.host_port] as $ports | ($ports | length) == ($ports | unique | length)' <<<"$services_json" >/dev/null \ + || die "the final state contains duplicate host-port ownership" +for node in node-a node-b; do + aws s3api head-object --bucket "$E2E_BUCKET" --key "cp/v1/nodes/$node.yaml" >/dev/null \ + || die "real S3 bucket has no rendered config for $node" +done +aws s3api list-objects-v2 --bucket "$E2E_BUCKET" --prefix cp/v1/ --output json > "$RUN_DIR/s3-inventory.json" +printf '%s\n' "$nodes_json" > "$RUN_DIR/nodes.json" +printf '%s\n' "$services_json" > "$RUN_DIR/services.json" +cp "$RUN_DIR/assets-manifest.json" "$RUN_DIR/asset-provenance.json" +SCENARIO_STATUS="passed" +log "local stateful two-node E2E passed" +log "final node placement: $(jq -r '.items[] | [.name, .node] | @tsv' <<<"$services_json" | tr '\n' ' ')" diff --git a/validation/e2e-local/scripts/validate.sh b/validation/e2e-local/scripts/validate.sh new file mode 100755 index 0000000..1aaabba --- /dev/null +++ b/validation/e2e-local/scripts/validate.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +GUEST_RUNNER="$SCRIPT_DIR/run-guest.sh" + +log() { + printf '==> %s\n' "$*" +} + +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "missing required command: $1" +} + +require_cmd aws +require_cmd curl +require_cmd git +require_cmd go +require_cmd jq +require_cmd make + +if [[ -z "${AWS_ACCESS_KEY_ID:-}" ]]; then + aws_profile="${AWS_PROFILE:-${FIREWORK_E2E_AWS_PROFILE:-artemnikitin}}" + eval "$(aws configure export-credentials --profile "$aws_profile" --format env)" + unset AWS_PROFILE +fi + +export AWS_REGION="${AWS_REGION:-${AWS_DEFAULT_REGION:-}}" +export AWS_DEFAULT_REGION="$AWS_REGION" +[[ -n "$AWS_REGION" ]] || die "AWS_REGION or AWS_DEFAULT_REGION is required" +aws sts get-caller-identity >/dev/null || die "AWS credentials are not usable" + +images_bucket="${FIREWORK_E2E_IMAGES_BUCKET:-artemnikitin-firework-images}" +[[ "$images_bucket" =~ ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ ]] \ + || die "FIREWORK_E2E_IMAGES_BUCKET is not a valid S3 bucket name" + +account_id="$(aws sts get-caller-identity --query Account --output text)" +run_id="$(date -u +%Y%m%dt%H%M%S)-$$" +bucket="firework-local-e2e-${account_id}-${run_id//[^a-zA-Z0-9-]/-}" +bucket="${bucket:0:63}" + +case "$(uname -s)" in + Darwin) + host_arch="arm64" + default_mode="lima" + ;; + Linux) + case "$(uname -m)" in + x86_64) host_arch="amd64" ;; + aarch64|arm64) host_arch="arm64" ;; + *) die "unsupported Linux architecture: $(uname -m)" ;; + esac + default_mode="linux" + ;; + *) die "unsupported host OS: $(uname -s)" ;; +esac +mode="${FIREWORK_E2E_MODE:-$default_mode}" +[[ "$mode" == linux || "$mode" == lima ]] || die "FIREWORK_E2E_MODE must be linux or lima" +if [[ "$mode" == linux && "$(uname -s)" != Linux ]]; then + die "FIREWORK_E2E_MODE=linux requires a Linux host" +fi +if [[ "$mode" == lima && "$(uname -s)" != Darwin ]]; then + die "FIREWORK_E2E_MODE=lima requires macOS" +fi + +if [[ "$mode" == lima ]]; then + [[ "$(uname -m)" == arm64 ]] || die "Lima mode requires an Apple Silicon host" + require_cmd limactl + log "host capability probe: macOS $(sw_vers -productVersion), Lima $(limactl --version | head -n 1)" + log "the guest probe will verify /dev/kvm, /dev/net/tun, namespaces, bridges, and iptables" +fi + +aws s3api head-object --bucket "$images_bucket" --key "${FIREWORK_E2E_ES_IMAGE_KEY:-tenant-2-elasticsearch-rootfs.ext4}" >/dev/null \ + || die "cannot read the Elasticsearch rootfs from s3://$images_bucket (set FIREWORK_E2E_IMAGES_BUCKET or image key overrides)" +aws s3api head-object --bucket "$images_bucket" --key "${FIREWORK_E2E_KIBANA_IMAGE_KEY:-tenant-2-kibana-rootfs.ext4}" >/dev/null \ + || die "cannot read the Kibana rootfs from s3://$images_bucket (set FIREWORK_E2E_IMAGES_BUCKET or image key overrides)" + +log "building Linux Firework binaries" +(cd "$REPO_ROOT" && make "build-linux-$host_arch" >/dev/null) + +commit="$(git -C "$REPO_ROOT" rev-parse HEAD)" +if [[ -n "${FIREWORK_E2E_WORKDIR:-}" ]]; then + host_run_dir="$FIREWORK_E2E_WORKDIR" +else + host_run_dir="$(mktemp -d "${TMPDIR:-/tmp}/firework-e2e-local.XXXXXX")" +fi +mkdir -p "$host_run_dir" + +conditional_key="probe/conditional-write-$run_id" +conditional_body="$host_run_dir/conditional-write.txt" +conditional_updated_body="$host_run_dir/conditional-write-updated.txt" +printf 'local-e2e-conditional-write\n' > "$conditional_body" +printf 'local-e2e-conditional-write-updated\n' > "$conditional_updated_body" + +run_conditional_write_probe() { + local first_etag second_etag + aws s3api put-object --bucket "$bucket" --key "$conditional_key" \ + --body "$conditional_body" --if-none-match '*' >/dev/null + if aws s3api put-object --bucket "$bucket" --key "$conditional_key" \ + --body "$conditional_body" --if-none-match '*' >/dev/null 2>&1; then + die "S3 If-None-Match conditional write unexpectedly overwrote an object" + fi + first_etag="$(aws s3api head-object --bucket "$bucket" --key "$conditional_key" --query ETag --output text)" + aws s3api put-object --bucket "$bucket" --key "$conditional_key" \ + --body "$conditional_updated_body" --if-match "$first_etag" >/dev/null + second_etag="$(aws s3api head-object --bucket "$bucket" --key "$conditional_key" --query ETag --output text)" + if aws s3api put-object --bucket "$bucket" --key "$conditional_key" \ + --body "$conditional_body" --if-match "$first_etag" >/dev/null 2>&1; then + die "S3 stale If-Match conditional write unexpectedly succeeded" + fi + jq -n --arg key "$conditional_key" --arg first "$first_etag" --arg second "$second_etag" \ + '{key:$key,if_none_match:"passed",if_match:"passed",stale_if_match:"passed",first_etag:$first,second_etag:$second}' \ + > "$host_run_dir/conditional-write.json" +} + +instance="" +guest_run_dir="/tmp/firework-e2e-local-$run_id" +status=1 +bucket_created=0 + +cleanup_bucket() { + local cleanup_status=0 + if [[ "$bucket_created" -ne 1 || "${FIREWORK_E2E_KEEP:-0}" == 1 ]]; then + return 0 + fi + log "deleting real S3 bucket $bucket" + aws s3 rm "s3://$bucket" --recursive >/dev/null 2>&1 || cleanup_status=1 + aws s3api delete-bucket --bucket "$bucket" >/dev/null 2>&1 || cleanup_status=1 + if [[ "$cleanup_status" -ne 0 ]]; then + printf 'WARNING: bucket cleanup failed; inspect and delete only %s manually\n' "$bucket" >&2 + fi + return "$cleanup_status" +} + +cleanup_runtime() { + set +e + if [[ "$mode" == lima && -n "$instance" ]]; then + if [[ "${FIREWORK_E2E_KEEP:-0}" == 1 ]]; then + limactl shell --yes "$instance" -- sudo -n chmod 644 "$guest_run_dir/manifest.json" >/dev/null 2>&1 || true + limactl copy "$instance:$guest_run_dir/manifest.json" "$host_run_dir/manifest.json" >/dev/null 2>&1 || true + else + limactl shell --yes "$instance" -- sudo -n chmod -R a+rX "$guest_run_dir" >/dev/null 2>&1 || true + limactl copy --recursive "$instance:$guest_run_dir" "$host_run_dir/guest" >/dev/null 2>&1 || true + fi + if [[ "${FIREWORK_E2E_KEEP:-0}" == 1 && -f "$host_run_dir/manifest.json" ]]; then + jq --arg instance "$instance" '. + {lima_instance:$instance}' \ + "$host_run_dir/manifest.json" > "$host_run_dir/manifest.tmp" + mv "$host_run_dir/manifest.tmp" "$host_run_dir/manifest.json" + fi + if [[ "${FIREWORK_E2E_KEEP:-0}" != 1 ]]; then + limactl stop --force "$instance" >/dev/null 2>&1 || true + limactl delete --force "$instance" >/dev/null 2>&1 || true + else + log "retaining Lima instance $instance" + fi + fi + cleanup_bucket || status=1 + if [[ "${FIREWORK_E2E_KEEP:-0}" == 1 ]]; then + log "retained host artifacts: $host_run_dir" + log "retained bucket: $bucket" + else + log "local E2E artifacts: $host_run_dir" + fi + exit "$status" +} +trap cleanup_runtime EXIT INT TERM + +log "creating real S3 bucket $bucket" +if [[ "$AWS_REGION" == us-east-1 ]]; then + aws s3api create-bucket --bucket "$bucket" --region "$AWS_REGION" >/dev/null +else + aws s3api create-bucket --bucket "$bucket" --region "$AWS_REGION" \ + --create-bucket-configuration "LocationConstraint=$AWS_REGION" >/dev/null +fi +bucket_created=1 +run_conditional_write_probe + +if [[ "$mode" == linux ]]; then + "$SCRIPT_DIR/check-env.sh" + sudo_env="AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_REGION,AWS_DEFAULT_REGION,AWS_EC2_METADATA_DISABLED,FIREWORK_E2E_KEEP,FIREWORK_E2E_TIMEOUT,FIREWORK_E2E_HEALTH_RETRIES,FIREWORK_E2E_IMAGES_BUCKET,FIREWORK_E2E_ES_IMAGE_KEY,FIREWORK_E2E_KIBANA_IMAGE_KEY,FIREWORK_E2E_FIRECRACKER_VERSION,FIREWORK_E2E_KERNEL_KEY,FIREWORK_E2E_SERVICE_VCPUS,FIREWORK_E2E_SERVICE_MEMORY_MB,FIREWORK_E2E_ES_VCPUS,FIREWORK_E2E_ES_MEMORY_MB,FIREWORK_E2E_KIBANA_VCPUS,FIREWORK_E2E_KIBANA_MEMORY_MB,FIREWORK_E2E_ES_JAVA_OPTS,FIREWORK_E2E_ENABLE_VOLUME,FIREWORK_E2E_VOLUME_SIZE,FIREWORK_E2E_STORAGE_CAPACITY" + if [[ "$(id -u)" -eq 0 ]]; then + bash "$GUEST_RUNNER" "$host_run_dir" "$bucket" "$AWS_REGION" "$commit" linux "$images_bucket" + else + require_cmd sudo + sudo --preserve-env="$sudo_env" bash "$GUEST_RUNNER" "$host_run_dir" "$bucket" "$AWS_REGION" "$commit" linux "$images_bucket" + fi + status=$? +else + require_cmd limactl + [[ "$(uname -m)" == arm64 ]] || die "Lima mode currently requires an Apple Silicon host" + + instance="firework-e2e-$run_id" + log "starting Lima instance $instance" + lima_cpus="${FIREWORK_E2E_LIMA_CPUS:-8}" + lima_memory="${FIREWORK_E2E_LIMA_MEMORY_GB:-12}" + lima_disk="${FIREWORK_E2E_LIMA_DISK_GB:-60}" + limactl start --yes --name "$instance" --plain --vm-type=vz --nested-virt \ + --cpus "$lima_cpus" --memory "$lima_memory" --disk "$lima_disk" >/dev/null + limactl shell --yes "$instance" -- sudo -n apt-get update -qq >/dev/null + limactl shell --yes "$instance" -- sudo -n apt-get install -y -qq awscli curl e2fsprogs git iproute2 iptables jq openssl >/dev/null + + guest_root="$guest_run_dir" + limactl shell --yes "$instance" -- mkdir -p "$guest_root/bin" "$guest_root/images" "$guest_root/scripts" + limactl copy "$GUEST_RUNNER" "$instance:$guest_root/run-guest.sh" + limactl copy "$SCRIPT_DIR/generate-pki.sh" "$instance:$guest_root/scripts/generate-pki.sh" + limactl copy "$SCRIPT_DIR/prepare-assets.sh" "$instance:$guest_root/scripts/prepare-assets.sh" + limactl copy "$SCRIPT_DIR/collect-diagnostics.sh" "$instance:$guest_root/scripts/collect-diagnostics.sh" + limactl copy "$REPO_ROOT/bin/firework-agent-linux-arm64" "$instance:$guest_root/bin/firework-agent-linux-arm64" + limactl copy "$REPO_ROOT/bin/firework-controlplane-linux-arm64" "$instance:$guest_root/bin/firework-controlplane-linux-arm64" + limactl copy "$REPO_ROOT/bin/fc-init-linux-arm64" "$instance:$guest_root/bin/fc-init-linux-arm64" + if [[ -n "${FIREWORK_E2E_FIRECRACKER_BIN:-}" ]]; then + [[ -x "$FIREWORK_E2E_FIRECRACKER_BIN" ]] || die "FIREWORK_E2E_FIRECRACKER_BIN is not executable" + limactl copy "$FIREWORK_E2E_FIRECRACKER_BIN" "$instance:$guest_root/bin/firecracker" + fi + if [[ -n "${FIREWORK_E2E_KERNEL:-}" ]]; then + [[ -r "$FIREWORK_E2E_KERNEL" ]] || die "FIREWORK_E2E_KERNEL is not readable" + limactl copy "$FIREWORK_E2E_KERNEL" "$instance:$guest_root/images/vmlinux" + fi + sudo_env="AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_REGION,AWS_DEFAULT_REGION,AWS_EC2_METADATA_DISABLED,FIREWORK_E2E_KEEP,FIREWORK_E2E_TIMEOUT,FIREWORK_E2E_HEALTH_RETRIES,FIREWORK_E2E_IMAGES_BUCKET,FIREWORK_E2E_ES_IMAGE_KEY,FIREWORK_E2E_KIBANA_IMAGE_KEY,FIREWORK_E2E_FIRECRACKER_VERSION,FIREWORK_E2E_KERNEL_KEY,FIREWORK_E2E_SERVICE_VCPUS,FIREWORK_E2E_SERVICE_MEMORY_MB,FIREWORK_E2E_ES_VCPUS,FIREWORK_E2E_ES_MEMORY_MB,FIREWORK_E2E_KIBANA_VCPUS,FIREWORK_E2E_KIBANA_MEMORY_MB,FIREWORK_E2E_ES_JAVA_OPTS,FIREWORK_E2E_ENABLE_VOLUME,FIREWORK_E2E_VOLUME_SIZE,FIREWORK_E2E_STORAGE_CAPACITY" + LIMA_SHELLENV_ALLOW="AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_REGION,AWS_DEFAULT_REGION,AWS_EC2_METADATA_DISABLED,FIREWORK_E2E_KEEP,FIREWORK_E2E_TIMEOUT,FIREWORK_E2E_HEALTH_RETRIES,FIREWORK_E2E_IMAGES_BUCKET,FIREWORK_E2E_ES_IMAGE_KEY,FIREWORK_E2E_KIBANA_IMAGE_KEY,FIREWORK_E2E_FIRECRACKER_VERSION,FIREWORK_E2E_KERNEL_KEY,FIREWORK_E2E_SERVICE_VCPUS,FIREWORK_E2E_SERVICE_MEMORY_MB,FIREWORK_E2E_ES_VCPUS,FIREWORK_E2E_ES_MEMORY_MB,FIREWORK_E2E_KIBANA_VCPUS,FIREWORK_E2E_KIBANA_MEMORY_MB,FIREWORK_E2E_ES_JAVA_OPTS,FIREWORK_E2E_ENABLE_VOLUME,FIREWORK_E2E_VOLUME_SIZE,FIREWORK_E2E_STORAGE_CAPACITY" \ + limactl shell --yes --preserve-env "$instance" -- \ + sudo -n --preserve-env="$sudo_env" env \ + FIREWORK_E2E_FIRECRACKER_BIN="$guest_root/images/firecracker" \ + FIREWORK_E2E_KERNEL="$guest_root/images/vmlinux" \ + FIREWORK_E2E_FIRECRACKER_VERSION="${FIREWORK_E2E_FIRECRACKER_VERSION:-1.12.0}" \ + FIREWORK_E2E_KERNEL_KEY="${FIREWORK_E2E_KERNEL_KEY:-firecracker-ci/v1.12/aarch64/vmlinux-5.10.233}" \ + FIREWORK_E2E_IMAGES_BUCKET="$images_bucket" \ + FIREWORK_E2E_ES_IMAGE_KEY="${FIREWORK_E2E_ES_IMAGE_KEY:-tenant-2-elasticsearch-rootfs.ext4}" \ + FIREWORK_E2E_KIBANA_IMAGE_KEY="${FIREWORK_E2E_KIBANA_IMAGE_KEY:-tenant-2-kibana-rootfs.ext4}" \ + FIREWORK_E2E_SCRIPT_DIR="$guest_root/scripts" \ + FIREWORK_E2E_ASSET_SCRIPT="$guest_root/scripts/prepare-assets.sh" \ + bash "$guest_root/run-guest.sh" "$guest_root" "$bucket" "$AWS_REGION" "$commit" lima "$images_bucket" + status=$? +fi