feat(schema): generate config JSON Schema from the provider registry

.github/workflows/ci.yml

@@ -93,6 +93,6 @@ jobs:
         with:
           go-version-file: go.mod
       - name: Install govulncheck
-        run: go install golang.org/x/vuln/cmd/govulncheck@v1.4.0
+        run: go install golang.org/x/vuln/cmd/govulncheck@v1.6.0
       - name: Run vulnerability gate
         run: ./scripts/govulncheck-gate.sh

.github/workflows/schemas.yml

@@ -0,0 +1,67 @@
+name: Schemas
+
+# Drift check for the generated schemas/ artifacts (committed in-tree,
+# consumed by nebari-docs). Runs schemagen and fails if the working tree
+# diverges from what was committed. Authors keep schemas/ in sync by
+# running `make schemas` locally before pushing.
+#
+# Paths-filtered to avoid running on PRs that can't affect schema output
+# (e.g. docs-only PRs, CI tweaks).
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'pkg/config/**'
+      - 'pkg/providers/**'
+      - 'pkg/nic/registry.go'
+      - 'pkg/nic/config_types.go'
+      - 'pkg/git/**'
+      - 'pkg/configschema/**'
+      - 'cmd/schemagen/**'
+      - 'schemas/**'
+      - 'go.mod'
+      - 'go.sum'
+      - 'Makefile'
+      - '.github/workflows/schemas.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'pkg/config/**'
+      - 'pkg/providers/**'
+      - 'pkg/nic/registry.go'
+      - 'pkg/nic/config_types.go'
+      - 'pkg/git/**'
+      - 'pkg/configschema/**'
+      - 'cmd/schemagen/**'
+      - 'schemas/**'
+      - 'go.mod'
+      - 'go.sum'
+      - 'Makefile'
+      - '.github/workflows/schemas.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  drift-check:
+    name: schemas/ drift check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+
+      - name: Regenerate schemas
+        run: make schemas
+
+      - name: Fail on drift
+        run: |
+          if ! git diff --exit-code schemas/; then
+            echo "::error::schemas/ is out of sync with the code. Run 'make schemas' locally and commit the result."
+            exit 1
+          fi

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help build test test-unit test-integration test-coverage test-race clean fmt vet lint vuln install pre-commit release-snapshot
+.PHONY: help build test test-unit test-integration test-coverage test-race clean fmt vet lint vuln install pre-commit release-snapshot schemas
 
 # Variables
 BINARY_NAME=nic
@@ -128,4 +128,9 @@ deps-update: ## Update Go dependencies
 	go mod tidy
 	@echo "Dependencies updated successfully"
 
+schemas: ## Regenerate JSON Schema documents under schemas/ from the Go config types
+	@echo "Regenerating schemas/..."
+	go run ./cmd/schemagen -out ./schemas
+	@echo "Schemas regenerated. Run 'git diff schemas/' to review."
+
 .DEFAULT_GOAL := help

cmd/schemagen/main.go

@@ -0,0 +1,239 @@
+// schemagen emits JSON Schema documents for nebari-config.yaml and each
+// registered provider's Config struct. It is an internal build/CI tool,
+// not a user-facing subcommand of nic.
+//
+// Output layout (default `-out ./schemas`):
+//
+//	schemas/
+//	  manifest.json
+//	  nebari-config.json
+//	  providers/
+//	    <name>.json    (one per registered cluster + DNS provider)
+//
+// The provider list comes from the nic registry (pkg/nic/registry.go) via
+// (*nic.Client).RegisteredConfigTypes, which reads each provider's config type
+// through the optional cluster.ConfigTyped / dns.ConfigTyped capability. There
+// is no parallel hard-coded list: registering a provider (that self-describes
+// its config type) extends the schemagen output automatically.
+//
+// Invocation: `make schemas` or `go run ./cmd/schemagen -out ./schemas`.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"strings"
+
+	"github.com/nebari-dev/nebari-infrastructure-core/pkg/config"
+	"github.com/nebari-dev/nebari-infrastructure-core/pkg/configschema"
+	"github.com/nebari-dev/nebari-infrastructure-core/pkg/nic"
+)
+
+func main() {
+	var (
+		outDir    string
+		providers string
+		pkgRoot   string
+		version   string
+	)
+	flag.StringVar(&outDir, "out", "./schemas", "output directory for generated schema files")
+	flag.StringVar(&providers, "providers", "", "comma-separated subset to regenerate (default: all registered)")
+	flag.StringVar(&pkgRoot, "pkg-root", "./pkg", "root directory whose Go packages are scanned for field godoc")
+	flag.StringVar(&version, "version", "", "version string stamped into manifest.json (default: empty)")
+	flag.Parse()
+
+	ctx := context.Background()
+	if err := run(ctx, outDir, providers, pkgRoot, version); err != nil {
+		slog.Error("schemagen failed", "error", err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context, outDir, providersFlag, pkgRoot, version string) error {
+	if err := os.MkdirAll(filepath.Join(outDir, "providers"), 0o750); err != nil {
+		return fmt.Errorf("mkdir %s: %w", outDir, err)
+	}
+
+	pkgPaths, err := collectPackagePaths(pkgRoot)
+	if err != nil {
+		return fmt.Errorf("collect package paths under %s: %w", pkgRoot, err)
+	}
+
+	client, err := nic.NewClient(ctx)
+	if err != nil {
+		return fmt.Errorf("build nic client: %w", err)
+	}
+	types := client.RegisteredConfigTypes(ctx)
+
+	filter := parseFilter(providersFlag)
+	emitTopLevel := len(filter) == 0
+
+	clusterNames := sortedKeys(types.Cluster)
+	dnsNames := sortedKeys(types.DNS)
+
+	if emitTopLevel {
+		if err := writeSchema(ctx, outDir, "nebari-config.json",
+			reflect.TypeFor[config.NebariConfig](),
+			"Nebari config", pkgPaths); err != nil {
+			return err
+		}
+	}
+
+	for _, name := range clusterNames {
+		if !accepts(filter, name) {
+			continue
+		}
+		if err := writeSchema(ctx, outDir, filepath.Join("providers", name+".json"),
+			types.Cluster[name],
+			fmt.Sprintf("%s cluster provider configuration", name), pkgPaths); err != nil {
+			return err
+		}
+	}
+
+	for _, name := range dnsNames {
+		if !accepts(filter, name) {
+			continue
+		}
+		if err := writeSchema(ctx, outDir, filepath.Join("providers", name+".json"),
+			types.DNS[name],
+			fmt.Sprintf("%s DNS provider configuration", name), pkgPaths); err != nil {
+			return err
+		}
+	}
+
+	if emitTopLevel {
+		if err := writeManifest(outDir, version, clusterNames, dnsNames); err != nil {
+			return err
+		}
+	}
+
+	fmt.Printf("schemagen wrote schemas under %s\n", outDir)
+	fmt.Printf("  cluster providers: %v\n", clusterNames)
+	fmt.Printf("  dns providers:     %v\n", dnsNames)
+	return nil
+}
+
+func writeSchema(ctx context.Context, outDir, relPath string, t reflect.Type, title string, pkgPaths []string) error {
+	data, err := configschema.Generate(ctx, t, configschema.FormatJSON, configschema.Options{
+		Title:        title,
+		PackagePaths: pkgPaths,
+	})
+	if err != nil {
+		return fmt.Errorf("generate %s: %w", relPath, err)
+	}
+	full := filepath.Join(outDir, relPath)
+	if err := os.WriteFile(full, data, 0o600); err != nil {
+		return fmt.Errorf("write %s: %w", full, err)
+	}
+	return nil
+}
+
+// manifest is the shape of schemas/manifest.json. The docs site fetches
+// this first to discover what schemas exist, then fetches each referenced
+// file. Adding a new provider extends Providers/DNS automatically.
+type manifest struct {
+	Comment   string   `json:"_comment"`
+	Version   string   `json:"version,omitempty"`
+	Providers []string `json:"providers"`
+	DNS       []string `json:"dns"`
+	TopLevel  string   `json:"top_level"`
+}
+
+// manifestComment marks the schemas/ tree as generated. JSON has no comment
+// syntax, so it rides as a "_comment" field the docs consumer ignores.
+const manifestComment = "Generated by cmd/schemagen (make schemas); do not edit by hand."
+
+func writeManifest(outDir, version string, cluster, dns []string) error {
+	m := manifest{
+		Comment:   manifestComment,
+		Version:   version,
+		Providers: cluster,
+		DNS:       dns,
+		TopLevel:  "nebari-config.json",
+	}
+	data, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal manifest: %w", err)
+	}
+	data = append(data, '\n')
+	return os.WriteFile(filepath.Join(outDir, "manifest.json"), data, 0o600)
+}
+
+// collectPackagePaths walks root and returns every subdirectory that
+// contains at least one non-test .go file. These paths are passed to
+// configschema.Generate as Options.PackagePaths so invopop/jsonschema
+// can pick up godoc comments wherever the type tree leads.
+func collectPackagePaths(root string) ([]string, error) {
+	var paths []string
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		if !d.IsDir() {
+			return nil
+		}
+		name := d.Name()
+		if strings.HasPrefix(name, ".") || name == "vendor" || name == "testdata" {
+			return fs.SkipDir
+		}
+		entries, err := os.ReadDir(path)
+		if err != nil {
+			return err
+		}
+		for _, e := range entries {
+			if e.IsDir() {
+				continue
+			}
+			n := e.Name()
+			if strings.HasSuffix(n, ".go") && !strings.HasSuffix(n, "_test.go") {
+				paths = append(paths, path)
+				return nil
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(paths)
+	return paths, nil
+}
+
+func sortedKeys(m map[string]reflect.Type) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+func parseFilter(raw string) map[string]struct{} {
+	if raw == "" {
+		return nil
+	}
+	out := make(map[string]struct{})
+	for name := range strings.SplitSeq(raw, ",") {
+		name = strings.TrimSpace(name)
+		if name != "" {
+			out[name] = struct{}{}
+		}
+	}
+	return out
+}
+
+func accepts(filter map[string]struct{}, name string) bool {
+	if filter == nil {
+		return true
+	}
+	_, ok := filter[name]
+	return ok
+}

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/goccy/go-yaml v1.19.2
 	github.com/hashicorp/terraform-exec v0.25.2
 	github.com/hashicorp/terraform-json v0.27.2
+	github.com/invopop/jsonschema v0.14.0
 	github.com/joho/godotenv v1.5.1
 	github.com/opentofu/tofudl v0.0.1
 	github.com/skeema/knownhosts v1.3.2
@@ -75,7 +76,9 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 // indirect
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/buger/jsonparser v1.1.2 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chai2010/gettext-go v1.0.3 // indirect
@@ -151,6 +154,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/pb33f/ordered-map/v2 v2.3.1 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pjbgf/sha1cd v0.6.0 // indirect
@@ -179,6 +183,7 @@ require (
 	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect
 	golang.org/x/net v0.56.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sync v0.21.0 // indirect