refactor(starters): render the starter workspaces from Go instead of shell

.github/workflows/starters.yml

@@ -4,15 +4,15 @@ on:
   pull_request:
     paths:
       - "starters/**"
-      - "scripts/gen-starters.sh"
+      - "cmd/starters/**"
       - "examples/local-config.yaml"
       - "examples/aws-config.yaml"
       - ".github/workflows/starters.yml"
   push:
     branches: [ main ]
     paths:
       - "starters/**"
-      - "scripts/gen-starters.sh"
+      - "cmd/starters/**"
       - "examples/local-config.yaml"
       - "examples/aws-config.yaml"
       - ".github/workflows/starters.yml"
@@ -34,7 +34,7 @@ env:
 
 jobs:
   validate-starters:
-    name: Validate starters
+    name: Validate
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
@@ -59,7 +59,7 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
-          ./scripts/gen-starters.sh dist/starters
+          go run ./cmd/starters -out dist/starters
 
       - name: Rendered starters are complete
         shell: bash
@@ -131,7 +131,7 @@ jobs:
     needs: validate-starters
     # Tag builds only, and deliberately NOT workflow_dispatch. A dispatch can
     # target any ref: from a branch it would publish starter-*:vmain (and pin a
-    # version from the PREVIOUS tag, since gen-starters.sh reads git describe),
+    # version from the PREVIOUS tag, since cmd/starters reads git describe),
     # and from an existing tag it would overwrite a released bundle - the exact
     # rewrite the trigger comment above says must never happen. Deployment-branch
     # rules live in repo settings and cannot be reviewed from this file, so the
@@ -188,7 +188,7 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
-          ./scripts/gen-starters.sh dist/starters
+          go run ./cmd/starters -out dist/starters
 
       - name: Configure quay registry
         shell: bash

Makefile

@@ -29,7 +29,7 @@ docs: ## Generate CLI and configuration reference documentation
 
 starters: ## Generate the Nebi starter workspaces into dist/starters
 	@echo "Generating starters..."
-	./scripts/gen-starters.sh dist/starters
+	go run ./cmd/starters -out dist/starters
 
 build-all: ## Build binaries for all platforms
 	@echo "Building for all platforms..."

cmd/starters/main.go

@@ -0,0 +1,123 @@
+// Command starters renders the Nebi starter workspaces from examples/.
+//
+// A starter is the provider's example config with the identity-bearing values
+// replaced by the CHANGEME sentinel, plus the pixi workspace that pins the
+// toolchain and the provider's README. examples/ stays the single source of
+// truth for config content, so there is no second copy to drift.
+//
+// Output is published as OCI bundles and deliberately not committed; see
+// .github/workflows/starters.yml.
+//
+// Usage:
+//
+//	go run ./cmd/starters -out dist/starters      # or: make starters
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+func main() {
+	outDir := flag.String("out", "dist/starters", "Output directory for the rendered starter workspaces")
+	nicVersion := flag.String("version", "", "nic version the starters pin (default: the most recent git tag, minus the leading v)")
+	templates := flag.String("templates", "starters/templates", "Directory holding pixi.toml.tmpl and the per-provider READMEs")
+	examples := flag.String("examples", "examples", "Directory holding <provider>-config.yaml")
+	flag.Parse()
+
+	version := *nicVersion
+	if version == "" {
+		v, err := latestTag()
+		if err != nil {
+			log.Fatalf("could not determine the nic version to pin: %v; pass -version", err)
+		}
+		version = v
+	}
+
+	if err := generate(*outDir, *templates, *examples, version); err != nil {
+		log.Fatalf("%v", err)
+	}
+}
+
+// latestTag returns the most recent tag with its leading v stripped. On a tag
+// build this is that tag; elsewhere it is the previous one, which is why the
+// publish workflow is tag-only.
+func latestTag() (string, error) {
+	out, err := exec.Command("git", "describe", "--tags", "--abbrev=0").Output()
+	if err != nil {
+		return "", fmt.Errorf("git describe: %w", err)
+	}
+	tag := strings.TrimSpace(string(out))
+	if tag == "" {
+		return "", fmt.Errorf("git describe returned nothing")
+	}
+	return strings.TrimPrefix(tag, "v"), nil
+}
+
+// generate renders every provider in scope. Failures accumulate: a restructure
+// of examples/ usually moves more than one key, and reporting them one run at
+// a time makes the author rediscover the same problem repeatedly.
+func generate(outDir, templates, examples, version string) error {
+	pixiTmpl, err := os.ReadFile(filepath.Clean(filepath.Join(templates, "pixi.toml.tmpl")))
+	if err != nil {
+		return fmt.Errorf("read pixi template: %w", err)
+	}
+
+	var problems []string
+	for _, name := range providerNames() {
+		if err := generateOne(outDir, templates, examples, name, version, pixiTmpl); err != nil {
+			problems = append(problems, fmt.Sprintf("%s: %v", name, err))
+			continue
+		}
+		fmt.Printf("generated %s (nic %s)\n", filepath.Join(outDir, name), version)
+	}
+
+	if len(problems) > 0 {
+		return fmt.Errorf("starter generation failed:\n  - %s", strings.Join(problems, "\n  - "))
+	}
+	return nil
+}
+
+func generateOne(outDir, templates, examples, name, version string, pixiTmpl []byte) error {
+	p := providers[name]
+
+	src, err := os.ReadFile(filepath.Clean(filepath.Join(examples, name+"-config.yaml")))
+	if err != nil {
+		return fmt.Errorf("read example: %w", err)
+	}
+
+	config, err := placeholderConfig(src, p.fields)
+	if err != nil {
+		return err
+	}
+
+	pixi, err := renderPixi(pixiTmpl, name, version, p.deps)
+	if err != nil {
+		return err
+	}
+
+	readme, err := os.ReadFile(filepath.Clean(filepath.Join(templates, "README."+name+".md")))
+	if err != nil {
+		return fmt.Errorf("read README: %w", err)
+	}
+
+	dest := filepath.Join(outDir, name)
+	if err := os.MkdirAll(dest, 0o750); err != nil {
+		return fmt.Errorf("create %s: %w", dest, err)
+	}
+	for file, content := range map[string][]byte{
+		"config.yaml": config,
+		"pixi.toml":   pixi,
+		"README.md":   readme,
+	} {
+		if err := os.WriteFile(filepath.Join(dest, file), content, 0o600); err != nil {
+			return fmt.Errorf("write %s: %w", file, err)
+		}
+	}
+	return nil
+}

cmd/starters/starters.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/parser"
+	"github.com/goccy/go-yaml/token"
+)
+
+// placeholder is the sentinel a starter ships instead of an identity-bearing
+// value. nic's validation rejects any config still containing it, so an
+// unedited starter cannot be deployed by accident; see
+// docs/operations/config-placeholders.md.
+const placeholder = "CHANGEME"
+
+// provider describes one starter: which config values the reader must supply,
+// and any conda dependency the provider needs beyond nic itself.
+type provider struct {
+	// fields are YAML paths, not line prefixes. A path either resolves to a
+	// value or it does not, so a key that moves, gets renamed, or gains a
+	// same-named sibling at another level is a hard error naming the path
+	// rather than something a text match can silently get wrong.
+	fields []string
+	deps   string
+}
+
+// providers is the set in scope. Extend deliberately: a new entry needs its
+// own placeholder paths, and generate refuses to emit a starter that declares
+// none rather than shipping one with every real value intact.
+var providers = map[string]provider{
+	"local": {
+		// kind runs everything locally: the certificate is self-signed and the
+		// GitOps repo is created for the user, so only the name is theirs.
+		fields: []string{"$.project_name"},
+		// The local provider drives kind through an embedded Go library, so
+		// there is no OpenTofu in this workspace.
+		deps: "",
+	},
+	"aws": {
+		fields: []string{
+			"$.project_name",
+			"$.domain",
+			"$.certificate.acme.email",
+			"$.repository.existing.url",
+			"$.repository.existing.path",
+		},
+		// Pinning OpenTofu is the point of a pinned toolchain: without it nic
+		// downloads an unpinned tofu at deploy time. The floor has to clear
+		// pkg/tofu.MinVersion - below that nic rejects the PATH binary and
+		// downloads one anyway, silently, which defeats the pin.
+		deps: `opentofu = ">=1.11.3,<2"`,
+	},
+}
+
+// providerNames returns the providers in scope, sorted, so output ordering is
+// deterministic regardless of map iteration order.
+func providerNames() []string {
+	names := make([]string, 0, len(providers))
+	for name := range providers {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	return names
+}
+
+// placeholderConfig rewrites src so that every path in fields carries the
+// placeholder instead of its real value, and returns the result.
+//
+// The edit is done on the source text rather than by marshalling the parsed
+// document back out: a round trip would reformat the file and drop the inline
+// comments that make the examples worth shipping. Each value's token gives the
+// line and the column where the value starts, so replacing from that column to
+// the end of the line touches nothing else, and a trailing comment on the same
+// line is reattached - those lines are exactly the ones whose hint the reader
+// needs most.
+func placeholderConfig(src []byte, fields []string) ([]byte, error) {
+	if len(fields) == 0 {
+		return nil, fmt.Errorf("no placeholder fields declared")
+	}
+
+	file, err := parser.ParseBytes(src, parser.ParseComments)
+	if err != nil {
+		return nil, fmt.Errorf("parse config: %w", err)
+	}
+
+	// Trailing newline handling: strings.Split on a file ending in "\n" yields
+	// a final empty element, which Join restores, so the byte count is stable.
+	lines := strings.Split(string(src), "\n")
+
+	for _, field := range fields {
+		path, err := yaml.PathString(field)
+		if err != nil {
+			return nil, fmt.Errorf("%s is not a valid YAML path: %w", field, err)
+		}
+
+		node, err := path.FilterFile(file)
+		if err != nil {
+			return nil, fmt.Errorf("%s resolved to nothing; the example has probably been restructured: %w", field, err)
+		}
+
+		tok := node.GetToken()
+		if tok == nil {
+			return nil, fmt.Errorf("%s has no source position", field)
+		}
+		// A block scalar's token is the |/> indicator, not the body, so the
+		// value spans lines the edit below would leave orphaned. Reject it by
+		// type: checking the token text for a newline never fires here.
+		if tok.Type == token.LiteralType || tok.Type == token.FoldedType || strings.Contains(tok.Value, "\n") {
+			return nil, fmt.Errorf("%s is a multi-line value; only single-line scalars can be placeholdered in place", field)
+		}
+
+		lineNo, col := tok.Position.Line, tok.Position.Column
+		if lineNo < 1 || lineNo > len(lines) {
+			return nil, fmt.Errorf("%s reports line %d, outside the file", field, lineNo)
+		}
+		line := lines[lineNo-1]
+		if col < 1 || col > len(line)+1 {
+			return nil, fmt.Errorf("%s reports column %d, outside line %d", field, col, lineNo)
+		}
+
+		rebuilt := line[:col-1] + placeholder
+		if c := node.GetComment(); c != nil {
+			if text := strings.TrimSpace(c.String()); text != "" {
+				rebuilt += "  " + text
+			}
+		}
+		lines[lineNo-1] = rebuilt
+	}
+
+	out := strings.Join(lines, "\n")
+
+	// Re-parse rather than trust the edit. A starter that no longer loads
+	// would still be "rejected" by nic validate, just for the wrong reason,
+	// and that failure is easy to mistake for the placeholder gate working.
+	if _, err := parser.ParseBytes([]byte(out), parser.ParseComments); err != nil {
+		return nil, fmt.Errorf("placeholdered config no longer parses: %w", err)
+	}
+	return []byte(out), nil
+}
+
+// renderPixi substitutes the workspace template's tokens. Kept as plain text
+// replacement because the template is a fixed file in this repo with three
+// tokens, not user input.
+func renderPixi(tmpl []byte, name, nicVersion, deps string) ([]byte, error) {
+	out := string(tmpl)
+	for token, value := range map[string]string{
+		"__PROVIDER__":      name,
+		"__NIC_VERSION__":   nicVersion,
+		"__PROVIDER_DEPS__": deps,
+	} {
+		out = strings.ReplaceAll(out, token, value)
+	}
+	if i := strings.Index(out, "__"); i != -1 {
+		return nil, fmt.Errorf("unsubstituted template token near %q", out[i:min(i+40, len(out))])
+	}
+	return []byte(out), nil
+}