diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 475192c..a29e170 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -6,7 +6,38 @@ on: pull_request: jobs: + validate-config: + # Validate the GitOps runtime config with the exact Firework enricher before + # any expensive image build. Pin FIREWORK_CONFIG_REF to the core version + # deployed to the nodes; do not validate against a floating main. + permissions: + contents: read + runs-on: ubuntu-24.04 + steps: + - name: Checkout GitOps config + uses: actions/checkout@v4 + + - name: Checkout Firework (pinned config contract) + uses: actions/checkout@v4 + with: + repository: artemnikitin/firework + ref: ${{ vars.FIREWORK_CONFIG_REF || 'be5c449ec546505cf03ca50a7118422a6139ee75' }} + path: .firework + token: ${{ secrets.FIREWORK_GITHUB_TOKEN || github.token }} + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: .firework/go.mod + + - name: Validate runtime config + working-directory: .firework + run: go run ./cmd/configcheck --require-remote-routing --input-dir "$GITHUB_WORKSPACE" + build: + needs: validate-config + permissions: + contents: read strategy: fail-fast: false matrix: @@ -53,3 +84,37 @@ jobs: AWS_REGION: ${{ vars.AWS_REGION }} S3_IMAGES_BUCKET: ${{ vars.S3_IMAGES_BUCKET }} run: make push + + - name: Save amd64 rootfs artifacts + if: matrix.target_platform == 'linux/amd64' + uses: actions/upload-artifact@v4 + with: + name: rootfs-amd64 + path: "*-rootfs.ext4" + if-no-files-found: error + + upload-gcs: + needs: build + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + name: rootfs-amd64 + + - name: Authenticate to GCP + uses: google-github-actions/auth@v3 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + + - name: Upload images to GCS + env: + GCS_IMAGES_BUCKET: ${{ vars.GCS_IMAGES_BUCKET }} + run: make push-gcs diff --git a/AGENTS.md b/AGENTS.md index c82f293..501421e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Project -This is the example GitOps input repo for Firework. It defines tenant service YAML and config overlays used to build Firecracker-ready rootfs images and publish them to S3. +This is the example GitOps input repo for Firework. It defines tenant service YAML and config overlays used to build Firecracker-ready rootfs images and publish them (ARM64 to S3, amd64 to GCS). Public routing is provider-neutral via `metadata.subdomain`; there is no provider-specific runtime config tree. ## Layout diff --git a/Makefile b/Makefile index 4bd4e04..ab1b620 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ TARGET_PLATFORM ?= linux/arm64 -.PHONY: build build-amd64 build-arm64 push +.PHONY: build build-amd64 build-arm64 push push-s3 push-gcs build: TARGET_PLATFORM="$(TARGET_PLATFORM)" bash ./scripts/build-images.sh @@ -13,3 +13,9 @@ build-arm64: push: bash ./scripts/push-images.sh + +push-s3: + S3_IMAGES_BUCKET="$(S3_IMAGES_BUCKET)" bash ./scripts/push-images.sh + +push-gcs: + GCS_IMAGES_BUCKET="$(GCS_IMAGES_BUCKET)" bash ./scripts/push-images.sh diff --git a/README.md b/README.md index a72eac1..ad17b86 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,12 @@ > This is an example deployment intended for demonstration and learning purposes only. It is not hardened, audited, etc. -Example GitOps repository for [Firework](https://github.com/artemnikitin/firework), focused on building Firecracker-ready rootfs images from Docker images and publishing them to S3. +Example GitOps repository for [Firework](https://github.com/artemnikitin/firework), focused on building Firecracker-ready rootfs images and publishing ARM64 images to S3 and amd64 images to GCS. ## Related Repositories - [firework](https://github.com/artemnikitin/firework) - orchestrator runtime (`firework-agent`, `enricher`, `scheduler`) -- [firework-deployment-example](https://github.com/artemnikitin/firework-deployment-example) - Terraform + Packer deployment on AWS +- [firework-deployment-example](https://github.com/artemnikitin/firework-deployment-example) - Terraform + Packer deployment on AWS and GCP ## Configuration Docs @@ -18,51 +18,17 @@ Service/config semantics are documented in the main `firework` repository: This repository intentionally keeps only high-level pipeline guidance. -## End-to-End Flow - -```mermaid -flowchart LR - GH[Git push to this repo] --> GHA[GitHub Actions build-images] - GH --> WEBHOOK[Webhook to enricher] - - GHA --> IMG[S3 images bucket
*-rootfs.ext4] - WEBHOOK --> ENRICHER[enricher Lambda] - ENRICHER --> CFG[S3 configs bucket
nodes/*.yaml] - - IMG --> AGENT[firework-agent nodes] - CFG --> AGENT - AGENT --> VM[Firecracker microVMs reconciled] -``` - ## CI Image Pipeline The `build-images` workflow runs on every pull request and every push to `main`. It builds the tenant rootfs images twice, once for `linux/arm64` and once for -`linux/amd64`, so a change fails CI if any declared `source_image` tag cannot be -exported for either architecture. The converter resolves the platform-specific -manifest digest with Docker buildx before creating the temporary container, so a -locally cached tag for another architecture cannot leak into the build. The -workflow installs CI dependencies, then delegates image work to the Makefile. -The Makefile is a thin entrypoint that calls the shell scripts in `scripts/`: +`linux/amd64`. -On pull requests, CI runs `make build` only for both architectures. On pushes to -`main`, CI builds both architectures and uploads the existing ARM64 artifact -names with `make push`. +### CI config validation -1. `make build` calls `scripts/build-images.sh`. -2. `scripts/build-images.sh` resolves `fc-init` (release asset, `go install`, - or bundled fallback build). -3. `scripts/build-images.sh` iterates over `tenants/*/*.yaml`. -4. `scripts/build-images.sh` reads `source_image` and optional `rootfs_size_mb` - from each tenant file. -5. `scripts/build-images.sh` creates `--rootfs.ext4` for the - requested `TARGET_PLATFORM` via `scripts/docker-to-rootfs.sh`. -6. `scripts/build-images.sh` applies config overlays in order (shared baseline - first, tenant-specific on top): - - `configs//` (shared baseline, applied first if present) - - `configs/-/` (tenant-specific, applied on top if present, overrides shared) -7. `make push` calls `scripts/push-images.sh` to upload resulting - `*-rootfs.ext4` artifacts to S3. +Before building images, the `validate-config` CI job runs Firework's +`cmd/configcheck --require-remote-routing` against this repository's root, using +the exact enricher of a pinned core version. Local platform-specific builds: diff --git a/scripts/push-images.sh b/scripts/push-images.sh index 8d079c9..81c2766 100644 --- a/scripts/push-images.sh +++ b/scripts/push-images.sh @@ -6,13 +6,19 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" cd "$REPO_ROOT" -if [ -z "${S3_IMAGES_BUCKET:-}" ]; then - echo "ERROR: S3_IMAGES_BUCKET is required" >&2 +if [ -n "${GCS_IMAGES_BUCKET:-}" ]; then + for ext4 in *-rootfs.ext4; do + [ -f "$ext4" ] || continue + echo "Uploading $ext4 to gs://${GCS_IMAGES_BUCKET}/${ext4}" + gcloud storage cp "$ext4" "gs://${GCS_IMAGES_BUCKET}/${ext4}" + done +elif [ -n "${S3_IMAGES_BUCKET:-}" ]; then + for ext4 in *-rootfs.ext4; do + [ -f "$ext4" ] || continue + echo "Uploading $ext4 to s3://${S3_IMAGES_BUCKET}/${ext4}" + aws s3 cp "$ext4" "s3://${S3_IMAGES_BUCKET}/${ext4}" + done +else + echo "ERROR: S3_IMAGES_BUCKET or GCS_IMAGES_BUCKET must be set" >&2 exit 1 fi - -for ext4 in *-rootfs.ext4; do - [ -f "$ext4" ] || continue - echo "Uploading $ext4 to s3://${S3_IMAGES_BUCKET}/${ext4}" - aws s3 cp "$ext4" "s3://${S3_IMAGES_BUCKET}/${ext4}" -done diff --git a/tenants/tenant-1/kibana.yaml b/tenants/tenant-1/kibana.yaml index f2c2b04..010fcf1 100644 --- a/tenants/tenant-1/kibana.yaml +++ b/tenants/tenant-1/kibana.yaml @@ -26,4 +26,4 @@ metadata: tenant: "tenant-1" tier: "standard" version: "8.19.11" - host: "tenant-1.artemnikitin.com" + subdomain: "tenant-1" diff --git a/tenants/tenant-2/kibana.yaml b/tenants/tenant-2/kibana.yaml index 82c3e25..6299a41 100644 --- a/tenants/tenant-2/kibana.yaml +++ b/tenants/tenant-2/kibana.yaml @@ -26,4 +26,4 @@ metadata: tenant: "tenant-2" tier: "premium" version: "9.3.0" - host: "tenant-2.artemnikitin.com" + subdomain: "tenant-2" diff --git a/tenants/tenant-3/elasticsearch-data-1.yaml b/tenants/tenant-3/elasticsearch-data-1.yaml index dbd3557..fcbf637 100644 --- a/tenants/tenant-3/elasticsearch-data-1.yaml +++ b/tenants/tenant-3/elasticsearch-data-1.yaml @@ -36,5 +36,5 @@ metadata: tenant: "tenant-3" tier: "premium" version: "8.17.10" - host: "es.artemnikitin.com" + subdomain: "es"