Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 13 additions & 69 deletions .github/workflows/build-images.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@ on:
- "tenants/**"
- "configs/**"
- "scripts/**"
- "Makefile"
- ".github/workflows/build-images.yaml"
pull_request:
paths:
- "tenants/**"
- "configs/**"
- "scripts/**"
- "Makefile"
- ".github/workflows/build-images.yaml"

jobs:
build:
Expand All @@ -30,82 +39,17 @@ jobs:
sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_arm64
sudo chmod +x /usr/local/bin/yq

- name: Resolve fc-init
- name: Build per-tenant rootfs images
env:
FC_INIT_VERSION: ${{ vars.FC_INIT_VERSION }}
FIREWORK_GITHUB_TOKEN: ${{ secrets.FIREWORK_GITHUB_TOKEN }}
run: |
set -euo pipefail
mkdir -p .cache/bin

if [ -n "${FC_INIT_VERSION:-}" ]; then
tag="${FC_INIT_VERSION#v}"
tag="v${tag}"
url="https://github.com/artemnikitin/firework/releases/download/${tag}/fc-init-linux-arm64"
echo "Downloading fc-init from release ${tag}"
curl -fsSL "$url" -o .cache/bin/fc-init
chmod +x .cache/bin/fc-init
else
echo "FC_INIT_VERSION not set; building fc-init from firework@main"
if [ -n "${FIREWORK_GITHUB_TOKEN:-}" ]; then
git config --global url."https://x-access-token:${FIREWORK_GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/"
export GOPRIVATE=github.com/artemnikitin/*
fi

if ! GOBIN="$PWD/.cache/bin" GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \
go install github.com/artemnikitin/firework/cmd/fc-init@main; then
echo "::warning::Failed to build fc-init from firework@main. Falling back to bundled source."
GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \
go build -ldflags "-s -w" -o .cache/bin/fc-init ./scripts/fc-init/main.go
fi
fi

- name: Build per-tenant rootfs images
run: |
chmod +x scripts/docker-to-rootfs.sh

for tenant_dir in tenants/*/; do
[ -d "$tenant_dir" ] || continue
tenant_id="$(basename "$tenant_dir")"
echo "::group::Tenant: $tenant_id"

for svc_file in "${tenant_dir}"*.yaml "${tenant_dir}"*.yml; do
[ -f "$svc_file" ] || continue
base_name="$(basename "${svc_file%.*}")" # e.g. "kibana"

source_image="$(yq '.source_image // ""' "$svc_file")"
if [ -z "$source_image" ]; then
echo "Skipping ${tenant_id}-${base_name} — no source_image"
continue
fi

size_mb="$(yq '.rootfs_size_mb // 512' "$svc_file")"
output="${tenant_id}-${base_name}-rootfs.ext4"

# Config overlay: tenant-specific overlay takes precedence.
overlay_arg=""
if [ -d "configs/${tenant_id}-${base_name}" ]; then
overlay_arg="configs/${tenant_id}-${base_name}"
elif [ -d "configs/${base_name}" ]; then
overlay_arg="configs/${base_name}"
fi

echo "Building $output from $source_image"
./scripts/docker-to-rootfs.sh "$source_image" "$output" "$size_mb" \
"${overlay_arg:-}" ".cache/bin/fc-init"
done
echo "::endgroup::"
done
run: make build

- name: Upload images to S3
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ vars.AWS_REGION }}
S3_IMAGES_BUCKET: ${{ vars.S3_IMAGES_BUCKET }}
run: |
for ext4 in *-rootfs.ext4; do
[ -f "$ext4" ] || continue
echo "Uploading $ext4 to s3://${S3_IMAGES_BUCKET}/${ext4}"
aws s3 cp "$ext4" "s3://${S3_IMAGES_BUCKET}/${ext4}"
done
run: make push
5 changes: 4 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,12 @@ This is the example GitOps input repo for Firework. It defines tenant service YA
## Layout

- `defaults.yaml`: global service defaults consumed by Firework enricher; changes here affect every service on the next enricher run.
- `Makefile`: image pipeline entrypoints used by CI (`build` and `push`).
- `tenants/<tenant>/<service>.yaml`: tenant service specs.
- `configs/<service>/` and `configs/<tenant>-<service>/`: rootfs overlays; tenant-specific overlays take precedence.
- `scripts/build-images.sh`: resolves `fc-init` and builds all tenant rootfs images.
- `scripts/docker-to-rootfs.sh`: converts Docker images into ext4 rootfs images.
- `scripts/push-images.sh`: uploads generated rootfs images to S3.
- `scripts/fc-init/`: fallback bundled `fc-init` source for CI.

## Conventions
Expand All @@ -27,6 +30,6 @@ For image pipeline changes, validate:
- `shellcheck scripts/docker-to-rootfs.sh`
- A targeted local rootfs build when Docker, `jq`, `mkfs.ext4`, and a linux/arm64 `fc-init` are available.

For CI-equivalent validation, refer to `.github/workflows/build-images.yaml`, but skip the step with uploading files to S3.
For CI-equivalent validation, run `make build`, but skip `make push`.

Do not upload to S3 or run cloud-mutating commands unless explicitly requested.
7 changes: 7 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
.PHONY: build push

build:
bash ./scripts/build-images.sh

push:
bash ./scripts/push-images.sh
31 changes: 21 additions & 10 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,24 @@ flowchart LR

## CI Image Pipeline

The `build-images` workflow does the following on relevant pushes:

1. Resolves `fc-init` (release asset, `go install`, or bundled fallback build).
2. Iterates over `tenants/*/*.yaml`.
3. Reads `source_image` and optional `rootfs_size_mb` from each tenant file.
4. Builds `<tenant>-<service>-rootfs.ext4` via `scripts/docker-to-rootfs.sh`.
5. Applies config overlays with precedence:
- `configs/<tenant>-<service>/` (tenant-specific)
- then `configs/<service>/` (shared)
6. Uploads resulting `*-rootfs.ext4` artifacts to S3.
The `build-images` workflow installs CI dependencies, then delegates image work to
the Makefile. The Makefile is a thin entrypoint that calls the shell scripts in
`scripts/`:

On pull requests, CI runs `make build` only. On pushes to `main`, CI runs both
`make build` and `make push`.

1. `make build` calls `scripts/build-images.sh`.
2. `scripts/build-images.sh` resolves `fc-init` (release asset, `go install`,
or bundled fallback build).
3. `scripts/build-images.sh` iterates over `tenants/*/*.yaml`.
4. `scripts/build-images.sh` reads `source_image` and optional `rootfs_size_mb`
from each tenant file.
5. `scripts/build-images.sh` creates `<tenant>-<service>-rootfs.ext4` via
`scripts/docker-to-rootfs.sh`.
6. `scripts/build-images.sh` applies config overlays in order (shared baseline
first, tenant-specific on top):
- `configs/<service>/` (shared baseline, applied first if present)
- `configs/<tenant>-<service>/` (tenant-specific, applied on top if present, overrides shared)
7. `make push` calls `scripts/push-images.sh` to upload resulting
`*-rootfs.ext4` artifacts to S3.
172 changes: 172 additions & 0 deletions scripts/build-images.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
#!/usr/bin/env bash

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$REPO_ROOT"

CACHE_BIN_DIR="${CACHE_BIN_DIR:-.cache/bin}"
if [[ "$CACHE_BIN_DIR" != /* ]]; then
CACHE_BIN_DIR="$REPO_ROOT/$CACHE_BIN_DIR"
fi

FC_INIT_BIN="${FC_INIT_BIN:-$CACHE_BIN_DIR/fc-init}"
if [[ "$FC_INIT_BIN" != /* ]]; then
FC_INIT_BIN="$REPO_ROOT/$FC_INIT_BIN"
fi

configure_tool_paths() {
if command -v mkfs.ext4 >/dev/null 2>&1; then
return
fi

if command -v brew >/dev/null 2>&1; then
local e2fsprogs_prefix
e2fsprogs_prefix="$(brew --prefix e2fsprogs 2>/dev/null || true)"
if [ -n "$e2fsprogs_prefix" ]; then
export PATH="${e2fsprogs_prefix}/sbin:${e2fsprogs_prefix}/bin:${PATH}"
fi
fi
}

install_fc_init_from_main() {
local go_path="$REPO_ROOT/.cache/go"
local installed_bin

mkdir -p "$go_path" "$(dirname "$FC_INIT_BIN")"

GOPATH="$go_path" GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \
go install github.com/artemnikitin/firework/cmd/fc-init@main

installed_bin="$go_path/bin/fc-init"
if [ ! -f "$installed_bin" ]; then
installed_bin="$go_path/bin/linux_arm64/fc-init"
fi

if [ ! -f "$installed_bin" ]; then
echo "ERROR: go install completed, but fc-init was not found in $go_path/bin" >&2
return 1
fi

cp "$installed_bin" "$FC_INIT_BIN"
chmod +x "$FC_INIT_BIN"
}

resolve_fc_init() {
mkdir -p "$CACHE_BIN_DIR"

if [ -n "${FC_INIT_VERSION:-}" ]; then
local tag
tag="${FC_INIT_VERSION#v}"
tag="v${tag}"

local url
url="https://github.com/artemnikitin/firework/releases/download/${tag}/fc-init-linux-arm64"
echo "Downloading fc-init from release ${tag}"
curl -fsSL "$url" -o "$FC_INIT_BIN"
chmod +x "$FC_INIT_BIN"
return
fi

echo "FC_INIT_VERSION not set; building fc-init from firework@main"
if [ -n "${FIREWORK_GITHUB_TOKEN:-}" ]; then
export GOPRIVATE='github.com/artemnikitin/*'
export GIT_CONFIG_COUNT=1
export GIT_CONFIG_KEY_0="url.https://x-access-token:${FIREWORK_GITHUB_TOKEN}@github.com/.insteadOf"
export GIT_CONFIG_VALUE_0="https://github.com/"
fi

if ! install_fc_init_from_main; then
echo "::warning::Failed to build fc-init from firework@main. Falling back to bundled source."
GOOS=linux GOARCH=arm64 CGO_ENABLED=0 \
go build -ldflags "-s -w" -o "$FC_INIT_BIN" ./scripts/fc-init/main.go
fi
}

overlay_arg_for() {
local tenant_id="$1"
local base_name="$2"

if [ -d "configs/${base_name}" ] && [ -d "configs/${tenant_id}-${base_name}" ]; then
printf '%s\n' "configs/${base_name}:configs/${tenant_id}-${base_name}"
elif [ -d "configs/${tenant_id}-${base_name}" ]; then
printf '%s\n' "configs/${tenant_id}-${base_name}"
elif [ -d "configs/${base_name}" ]; then
printf '%s\n' "configs/${base_name}"
fi
}

yaml_value() {
local file="$1"
local key="$2"
local default_value="$3"

if command -v yq >/dev/null 2>&1; then
yq ".${key} // \"${default_value}\"" "$file"
return
fi

if command -v ruby >/dev/null 2>&1; then
ruby -ryaml -e '
file, key, default_value = ARGV
data = YAML.load_file(file) || {}
value = data[key]
print(value.nil? ? default_value : value)
' "$file" "$key" "$default_value"
return
fi

awk -v key="$key" -v default_value="$default_value" '
BEGIN { value = default_value }
$0 ~ "^[[:space:]]*" key ":" {
sub("^[[:space:]]*" key ":[[:space:]]*", "")
gsub(/^"|"$/, "")
value = $0
exit
}
END { print value }
' "$file"
}

build_images() {
for tenant_dir in tenants/*/; do
[ -d "$tenant_dir" ] || continue
local tenant_id
tenant_id="$(basename "$tenant_dir")"
echo "::group::Tenant: $tenant_id"

for svc_file in "${tenant_dir}"*.yaml "${tenant_dir}"*.yml; do
[ -f "$svc_file" ] || continue

local base_name
base_name="$(basename "${svc_file%.*}")"

local source_image
source_image="$(yaml_value "$svc_file" source_image "")"
if [ -z "$source_image" ]; then
echo "Skipping ${tenant_id}-${base_name} - no source_image"
continue
fi

local size_mb
size_mb="$(yaml_value "$svc_file" rootfs_size_mb 512)"

local output
output="${tenant_id}-${base_name}-rootfs.ext4"

local overlay_arg
overlay_arg="$(overlay_arg_for "$tenant_id" "$base_name")"

echo "Building $output from $source_image"
bash ./scripts/docker-to-rootfs.sh "$source_image" "$output" "$size_mb" \
"${overlay_arg:-}" "$FC_INIT_BIN"
done

echo "::endgroup::"
done
}

configure_tool_paths
resolve_fc_init
build_images
23 changes: 15 additions & 8 deletions scripts/docker-to-rootfs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
#
# Usage: ./scripts/docker-to-rootfs.sh <docker-image> <output.ext4> [size_mb] [overlay_dir] [fc_init_bin]
#
# overlay_dir Optional directory whose contents are copied into the rootfs,
# mirroring the guest filesystem layout. For example, placing a
# file at overlay_dir/usr/share/elasticsearch/config/elasticsearch.yml
# overwrites that path in the rootfs.
# overlay_dir Optional colon-separated list of directories whose contents are
# copied into the rootfs in order, mirroring the guest filesystem
# layout. Later directories override earlier ones, so pass the shared
# baseline first and tenant-specific overlay second. For example:
# configs/elasticsearch:configs/tenant-1-elasticsearch
# fc_init_bin Optional path to a prebuilt linux/arm64 fc-init binary.
# If omitted, the script tries:
# 1) FC_INIT_BIN env var
Expand Down Expand Up @@ -164,10 +165,16 @@ RUNTIME_WRITABLE_PATHS_JSON="$(jq -cn \
| unique
')"

# Apply config overlay if provided.
if [ -n "$OVERLAY_DIR" ] && [ -d "$OVERLAY_DIR" ]; then
echo "==> Applying config overlay from $OVERLAY_DIR"
cp -r "$OVERLAY_DIR/." "$ROOTFS/"
# Apply config overlays in order. OVERLAY_DIR may be a colon-separated list;
# later entries override earlier ones (shared baseline first, tenant-specific second).
if [ -n "$OVERLAY_DIR" ]; then
IFS=: read -ra overlay_dirs <<< "$OVERLAY_DIR"
for dir in "${overlay_dirs[@]}"; do
[ -n "$dir" ] || continue
[ -d "$dir" ] || continue
echo "==> Applying config overlay from $dir"
cp -r "$dir/." "$ROOTFS/"
done
fi

# Some upstream images carry runtime-generated files that should not be baked
Expand Down
18 changes: 18 additions & 0 deletions scripts/push-images.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#!/usr/bin/env bash

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$REPO_ROOT"

if [ -z "${S3_IMAGES_BUCKET:-}" ]; then
echo "ERROR: S3_IMAGES_BUCKET is required" >&2
exit 1
fi

for ext4 in *-rootfs.ext4; do
[ -f "$ext4" ] || continue
echo "Uploading $ext4 to s3://${S3_IMAGES_BUCKET}/${ext4}"
aws s3 cp "$ext4" "s3://${S3_IMAGES_BUCKET}/${ext4}"
done
Loading