From 637104bafcaa2e710bc800d7237385707a50f0bb Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Fri, 25 Sep 2026 16:37:37 -0700 Subject: [PATCH] Add image override variables for third-party services Postgres, OpenBao, Redis, DynamoDB Local and smtp4dev were literal tags that float, so a benchmark host could not pin them without a compose override file. Each now reads a _IMAGE variable, following the existing PIRI_IMAGE and MINIO_IMAGE convention, with the current tag as the default. With the variables unset, docker compose config renders byte-identical output. The generator emits the same form for piri-postgres and piri-postgres-init, and .env lists the new variables. Tests pin each service to its variable and fail if any image in the root stack lacks an override variable. Co-Authored-By: Claude Opus 5.5 --- .env | 8 ++ images_test.go | 125 +++++++++++++++++++++++++++ pkg/generate/compose.go | 4 +- pkg/generate/generate_test.go | 24 +++++ systems/common/compose.yml | 4 +- systems/hilt/compose.yml | 4 +- systems/indexing/indexer/compose.yml | 2 +- systems/ingot/compose.yml | 6 +- systems/plc/compose.yml | 2 +- systems/swarf/compose.yml | 2 +- systems/upload/compose.yml | 2 +- 11 files changed, 170 insertions(+), 13 deletions(-) create mode 100644 images_test.go diff --git a/.env b/.env index 5e9f76e..884edb3 100644 --- a/.env +++ b/.env @@ -24,3 +24,11 @@ # Infrastructure #BLOCKCHAIN_IMAGE= + +# Third-party +#POSTGRES_IMAGE= +#OPENBAO_IMAGE= +#REDIS_IMAGE= +#DYNAMODB_LOCAL_IMAGE= +#SMTP4DEV_IMAGE= +#MINIO_IMAGE= diff --git a/images_test.go b/images_test.go new file mode 100644 index 0000000..e5d8a2a --- /dev/null +++ b/images_test.go @@ -0,0 +1,125 @@ +package smelt_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// Third-party images read a _IMAGE variable whose default is the tag +// the stack used before the variable existed, so an unset variable renders +// the same compose config and a set one repins exactly these services. +var thirdPartyImages = map[string]map[string]string{ + "systems/ingot/compose.yml": { + "ingot-openbao": "${OPENBAO_IMAGE:-openbao/openbao:2.6}", + "ingot-openbao-init": "${OPENBAO_IMAGE:-openbao/openbao:2.6}", + "ingot-postgres": "${POSTGRES_IMAGE:-postgres:16-alpine}", + }, + "systems/upload/compose.yml": { + "postgres": "${POSTGRES_IMAGE:-postgres:16-alpine}", + }, + "systems/hilt/compose.yml": { + "hilt-vault": "${OPENBAO_IMAGE:-openbao/openbao:2.6}", + "hilt-postgres": "${POSTGRES_IMAGE:-postgres:16-alpine}", + }, + "systems/swarf/compose.yml": { + "swarf-postgres": "${POSTGRES_IMAGE:-postgres:16-alpine}", + }, + "systems/plc/compose.yml": { + "plc-postgres": "${POSTGRES_IMAGE:-postgres:16-alpine}", + }, + "systems/common/compose.yml": { + "dynamodb-local": "${DYNAMODB_LOCAL_IMAGE:-amazon/dynamodb-local:latest}", + "email": "${SMTP4DEV_IMAGE:-rnwood/smtp4dev:v3}", + }, + "systems/indexing/indexer/compose.yml": { + "redis": "${REDIS_IMAGE:-redis:7-alpine}", + }, +} + +type composeImages struct { + Services map[string]struct { + Image string `yaml:"image"` + } `yaml:"services"` +} + +func readComposeImages(t *testing.T, path string) composeImages { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var c composeImages + if err := yaml.Unmarshal(data, &c); err != nil { + t.Fatalf("%s: %v", path, err) + } + return c +} + +func TestThirdPartyImagesReadOverrideVariables(t *testing.T) { + for path, want := range thirdPartyImages { + c := readComposeImages(t, path) + for svc, image := range want { + got, ok := c.Services[svc] + if !ok { + t.Errorf("%s: service %q not found", path, svc) + continue + } + if got.Image != image { + t.Errorf("%s: %s image = %q, want %q", path, svc, got.Image, image) + } + } + } +} + +// Every image the root compose file runs is pinnable from the environment. +// The telemetry system is not part of the root stack and keeps its literals. +func TestIncludedComposeImagesAreInterpolated(t *testing.T) { + root := readIncludes(t, "compose.yml") + checked := 0 + for _, path := range root { + if strings.HasPrefix(path, "generated/") { + continue // covered by the generator's own tests + } + for _, p := range append([]string{path}, readIncludes(t, path)...) { + for svc, s := range readComposeImages(t, p).Services { + if s.Image == "" { + continue // built locally + } + checked++ + if !strings.HasPrefix(s.Image, "${") { + t.Errorf("%s: %s image %q has no override variable", p, svc, s.Image) + } + } + } + } + if checked == 0 { + t.Fatal("no images checked") + } +} + +// readIncludes returns the include paths of a compose file, relative to the +// repository root. +func readIncludes(t *testing.T, path string) []string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var c struct { + Include []struct { + Path string `yaml:"path"` + } `yaml:"include"` + } + if err := yaml.Unmarshal(data, &c); err != nil { + t.Fatalf("%s: %v", path, err) + } + var out []string + for _, inc := range c.Include { + out = append(out, filepath.Join(filepath.Dir(path), inc.Path)) + } + return out +} diff --git a/pkg/generate/compose.go b/pkg/generate/compose.go index 6979a81..00a8c1b 100644 --- a/pkg/generate/compose.go +++ b/pkg/generate/compose.go @@ -139,7 +139,7 @@ func buildPiriService(node manifest.ResolvedPiriNode) ComposeService { func buildPostgresService() ComposeService { return ComposeService{ - Image: "postgres:16-alpine", + Image: "${POSTGRES_IMAGE:-postgres:16-alpine}", Ports: []string{"${SMELT_PIRI_POSTGRES_PORT:-15074:5432}"}, Environment: []string{ "POSTGRES_USER=piri", @@ -164,7 +164,7 @@ func buildPostgresService() ComposeService { func buildPostgresInitService(databases []string) ComposeService { return ComposeService{ - Image: "postgres:16-alpine", + Image: "${POSTGRES_IMAGE:-postgres:16-alpine}", Entrypoint: []string{"sh", "/postgres-init.sh"}, Environment: []string{ "PGPASSWORD=piri", diff --git a/pkg/generate/generate_test.go b/pkg/generate/generate_test.go index a669ed1..cd083c7 100644 --- a/pkg/generate/generate_test.go +++ b/pkg/generate/generate_test.go @@ -185,6 +185,30 @@ func TestGeneratePiriComposeMultiNode(t *testing.T) { } } +func TestGeneratePiriComposePostgresImageOverride(t *testing.T) { + nodes := []manifest.ResolvedPiriNode{ + {Name: "piri-0", Index: 0, Storage: manifest.StorageSpec{DB: "postgres", Blob: "filesystem"}}, + } + + data, err := GeneratePiriCompose(nodes) + if err != nil { + t.Fatal(err) + } + + var compose ComposeFile + if err := yaml.Unmarshal(data, &compose); err != nil { + t.Fatalf("unmarshal generated compose: %v", err) + } + + // Unset, POSTGRES_IMAGE falls back to the tag used before it existed. + const want = "${POSTGRES_IMAGE:-postgres:16-alpine}" + for _, name := range []string{"piri-postgres", "piri-postgres-init"} { + if got := compose.Services[name].Image; got != want { + t.Errorf("%s image = %q, want %q", name, got, want) + } + } +} + func TestGeneratePiriComposeSerializedStartup(t *testing.T) { nodes := []manifest.ResolvedPiriNode{ {Name: "piri-0", Index: 0, Storage: manifest.StorageSpec{DB: "sqlite", Blob: "filesystem"}}, diff --git a/systems/common/compose.yml b/systems/common/compose.yml index 76c00ce..5e40836 100644 --- a/systems/common/compose.yml +++ b/systems/common/compose.yml @@ -11,7 +11,7 @@ services: dynamodb-local: - image: amazon/dynamodb-local:latest + image: ${DYNAMODB_LOCAL_IMAGE:-amazon/dynamodb-local:latest} # Root so docker's auto-created volume dir (root-owned) is writable. The # upstream image's dynamodblocal uid 1000 can't write into a fresh named # volume without a chown sidecar. @@ -69,7 +69,7 @@ services: networks: - forge-network email: - image: rnwood/smtp4dev:v3 + image: ${SMTP4DEV_IMAGE:-rnwood/smtp4dev:v3} ports: - "${SMELT_SMTP_PORT:-15080:25}" - "${SMELT_SMTP_WEB_PORT:-15081:80}" # Web UI / API diff --git a/systems/hilt/compose.yml b/systems/hilt/compose.yml index be5ee01..4e3c34f 100644 --- a/systems/hilt/compose.yml +++ b/systems/hilt/compose.yml @@ -107,7 +107,7 @@ services: # restarts but not a hilt-vault container restart. The image's default # command is `server -dev`; the entrypoint reads the BAO_DEV_* vars below. hilt-vault: - image: openbao/openbao:2.6 + image: ${OPENBAO_IMAGE:-openbao/openbao:2.6} ports: - "${SMELT_HILT_VAULT_PORT:-15112:8200}" environment: @@ -129,7 +129,7 @@ services: # enough — no init sidecar needed. Named hilt-postgres because `postgres` # is taken by upload's sidecar. hilt-postgres: - image: postgres:16-alpine + image: ${POSTGRES_IMAGE:-postgres:16-alpine} ports: - "${SMELT_HILT_POSTGRES_PORT:-15111:5432}" environment: diff --git a/systems/indexing/indexer/compose.yml b/systems/indexing/indexer/compose.yml index 5c46fde..a539c75 100644 --- a/systems/indexing/indexer/compose.yml +++ b/systems/indexing/indexer/compose.yml @@ -6,7 +6,7 @@ services: # Redis - Cache backend for indexing service redis: - image: redis:7-alpine + image: ${REDIS_IMAGE:-redis:7-alpine} ports: - "${SMELT_REDIS_PORT:-15020:6379}" healthcheck: diff --git a/systems/ingot/compose.yml b/systems/ingot/compose.yml index f74a54f..6efd40a 100644 --- a/systems/ingot/compose.yml +++ b/systems/ingot/compose.yml @@ -79,7 +79,7 @@ services: # and a restarted one is sealed until ingot-openbao-init runs again (which # `make up` does, since one-shot services re-run on every up). ingot-openbao: - image: openbao/openbao:2.6 + image: ${OPENBAO_IMAGE:-openbao/openbao:2.6} command: ["server"] ports: - "${SMELT_INGOT_OPENBAO_PORT:-15132:8200}" @@ -107,7 +107,7 @@ services: # the ingot policy (encrypt/decrypt/rewrap on that key only), and ingot's # scoped token exist. See openbao/init.sh. ingot-openbao-init: - image: openbao/openbao:2.6 + image: ${OPENBAO_IMAGE:-openbao/openbao:2.6} user: "0:0" # the image runs as `openbao`; the init volume is root-owned entrypoint: ["sh", "/init.sh"] environment: @@ -128,7 +128,7 @@ services: # migrations under the `ingot` schema on startup; the `ingot` database # created by POSTGRES_DB is enough. ingot-postgres: - image: postgres:16-alpine + image: ${POSTGRES_IMAGE:-postgres:16-alpine} ports: - "${SMELT_INGOT_POSTGRES_PORT:-15131:5432}" environment: diff --git a/systems/plc/compose.yml b/systems/plc/compose.yml index b143476..c717bd1 100644 --- a/systems/plc/compose.yml +++ b/systems/plc/compose.yml @@ -38,7 +38,7 @@ services: # Postgres for the PLC operation log. The server runs kysely migrations on # startup, so the empty `plc` database created by POSTGRES_DB is enough. plc-postgres: - image: postgres:16-alpine + image: ${POSTGRES_IMAGE:-postgres:16-alpine} ports: - "${SMELT_PLC_POSTGRES_PORT:-15121:5432}" environment: diff --git a/systems/swarf/compose.yml b/systems/swarf/compose.yml index 6560b78..a9d6c4b 100644 --- a/systems/swarf/compose.yml +++ b/systems/swarf/compose.yml @@ -47,7 +47,7 @@ services: # Postgres for swarf's revocation store. Swarf runs goose migrations on # startup, so the `swarf` database created by POSTGRES_DB is enough. swarf-postgres: - image: postgres:16-alpine + image: ${POSTGRES_IMAGE:-postgres:16-alpine} ports: - "${SMELT_SWARF_POSTGRES_PORT:-15141:5432}" environment: diff --git a/systems/upload/compose.yml b/systems/upload/compose.yml index 1c3872f..67fa05e 100644 --- a/systems/upload/compose.yml +++ b/systems/upload/compose.yml @@ -74,7 +74,7 @@ services: # POSTGRES_DB is enough. Separate from `piri-postgres` (which is generated # from smelt.yml when piri nodes use db: postgres). postgres: - image: postgres:16-alpine + image: ${POSTGRES_IMAGE:-postgres:16-alpine} ports: - "${SMELT_UPLOAD_POSTGRES_PORT:-15061:5432}" environment: