From 136250a294e51836447ca2dab146d2bd4e07a3b9 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 5 Mar 2026 11:20:00 +0000 Subject: [PATCH 1/8] chore: First small step to HA --- Makefile | 8 + charts/gitops-reverser/README.md | 5 + .../gitops-reverser/templates/deployment.yaml | 12 ++ charts/gitops-reverser/values.yaml | 16 ++ cmd/main.go | 54 ++++++ cmd/main_audit_server_test.go | 33 ++++ config/deployment.yaml | 3 + docs/design/audit-webhook-redis-queue-plan.md | 181 ++++++++++++++++++ go.mod | 5 + go.sum | 10 + internal/queue/redis_audit_queue.go | 170 ++++++++++++++++ internal/queue/redis_audit_queue_test.go | 85 ++++++++ internal/webhook/audit_handler.go | 14 ++ internal/webhook/audit_handler_test.go | 60 ++++++ test/e2e/audit_redis_e2e_test.go | 121 ++++++++++++ 15 files changed, 777 insertions(+) create mode 100644 docs/design/audit-webhook-redis-queue-plan.md create mode 100644 internal/queue/redis_audit_queue.go create mode 100644 internal/queue/redis_audit_queue_test.go create mode 100644 test/e2e/audit_redis_e2e_test.go diff --git a/Makefile b/Makefile index 4672f57f6..8042addaf 100644 --- a/Makefile +++ b/Makefile @@ -584,6 +584,14 @@ test-e2e-quickstart-helm: ## Run quickstart smoke test (Helm install) export E2E_AGE_KEY_FILE=$(CS)/age-key.txt go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=quickstart-framework +.PHONY: test-e2e-audit-redis +test-e2e-audit-redis: ## Run dedicated audit->Redis enqueue e2e scenario + export CTX=$(CTX) + export INSTALL_MODE=$(INSTALL_MODE) + export NAMESPACE=$(NAMESPACE) + export E2E_AGE_KEY_FILE=$(CS)/age-key.txt + go test ./test/e2e/ -v -ginkgo.v -ginkgo.label-filter=audit-redis + .PHONY: test-e2e-quickstart-manifest test-e2e-quickstart-manifest: ## Run quickstart smoke test (manifest install) export CTX=$(CTX) diff --git a/charts/gitops-reverser/README.md b/charts/gitops-reverser/README.md index e2121074e..9c3031e2a 100644 --- a/charts/gitops-reverser/README.md +++ b/charts/gitops-reverser/README.md @@ -205,6 +205,11 @@ webhook: | `servers.audit.timeouts.write` | Audit-server write timeout | `30s` | | `servers.audit.timeouts.idle` | Audit-server idle timeout | `60s` | | `servers.audit.tls.secretName` | Secret name for audit TLS cert/key | `-audit-server-cert` | +| `queue.redis.enabled` | Enqueue accepted audit events to Redis Streams | `false` | +| `queue.redis.addr` | Redis endpoint for durable audit queueing | `""` | +| `queue.redis.stream` | Redis stream name for audit events | `gitopsreverser.audit.events.v1` | +| `queue.redis.maxLen` | Approximate stream max length (`0` disables trim) | `0` | +| `queue.redis.tls.enabled` | Enable TLS for Redis connection | `false` | | `servers.metrics.bindAddress` | Metrics listener bind address | `:8080` | | `servers.metrics.tls.enabled` | Serve metrics with TLS | `false` | | `servers.metrics.tls.certPath` | Metrics TLS certificate mount path | `/tmp/k8s-metrics-server/metrics-server-certs` | diff --git a/charts/gitops-reverser/templates/deployment.yaml b/charts/gitops-reverser/templates/deployment.yaml index ad02d7813..cd9a4daff 100644 --- a/charts/gitops-reverser/templates/deployment.yaml +++ b/charts/gitops-reverser/templates/deployment.yaml @@ -69,6 +69,18 @@ spec: - --audit-read-timeout={{ .Values.servers.audit.timeouts.read }} - --audit-write-timeout={{ .Values.servers.audit.timeouts.write }} - --audit-idle-timeout={{ .Values.servers.audit.timeouts.idle }} + {{- if .Values.queue.redis.enabled }} + - --audit-redis-enabled + - --audit-redis-addr={{ .Values.queue.redis.addr }} + - --audit-redis-username={{ .Values.queue.redis.username | quote }} + - --audit-redis-password={{ .Values.queue.redis.password | quote }} + - --audit-redis-db={{ .Values.queue.redis.db }} + - --audit-redis-stream={{ .Values.queue.redis.stream }} + - --audit-redis-max-len={{ .Values.queue.redis.maxLen }} + {{- if .Values.queue.redis.tls.enabled }} + - --audit-redis-tls + {{- end }} + {{- end }} {{- if .Values.logging.level }} - --zap-log-level={{ .Values.logging.level }} {{- end }} diff --git a/charts/gitops-reverser/values.yaml b/charts/gitops-reverser/values.yaml index 82b83c106..a712c75f7 100644 --- a/charts/gitops-reverser/values.yaml +++ b/charts/gitops-reverser/values.yaml @@ -128,6 +128,22 @@ webhook: # Object selector for validating webhook objectSelector: {} +# Durable audit queue configuration. +queue: + redis: + # Enable Redis Stream enqueueing for accepted audit events. + enabled: false + # Redis endpoint in host:port format. + addr: "" + username: "" + password: "" + db: 0 + stream: "gitopsreverser.audit.events.v1" + # Approximate max stream length; 0 means no trimming. + maxLen: 0 + tls: + enabled: false + # TLS certificate management (Kubernetes requires this for API server callbacks) certificates: name: diff --git a/cmd/main.go b/cmd/main.go index 0828ffa05..8e0aa38e8 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -52,6 +52,7 @@ import ( "github.com/ConfigButler/gitops-reverser/internal/controller" "github.com/ConfigButler/gitops-reverser/internal/correlation" "github.com/ConfigButler/gitops-reverser/internal/git" + "github.com/ConfigButler/gitops-reverser/internal/queue" "github.com/ConfigButler/gitops-reverser/internal/reconcile" "github.com/ConfigButler/gitops-reverser/internal/rulestore" "github.com/ConfigButler/gitops-reverser/internal/telemetry" @@ -204,9 +205,29 @@ func main() { setupLog.Info("GitTarget validator webhook registered - enforcing uniqueness constraint") // Register experimental audit webhook for metrics collection + var auditQueue webhookhandler.AuditEventQueue + if cfg.auditRedisEnabled { + auditQueue, err = queue.NewRedisAuditQueue(queue.RedisAuditQueueConfig{ + Addr: cfg.auditRedisAddr, + Username: cfg.auditRedisUsername, + AuthValue: cfg.auditRedisPassword, + DB: cfg.auditRedisDB, + Stream: cfg.auditRedisStream, + MaxLen: cfg.auditRedisMaxLen, + TLSEnabled: cfg.auditRedisTLS, + }) + fatalIfErr(err, "unable to initialize audit redis queue") + setupLog.Info("Audit Redis queue enabled", + "redis-address", cfg.auditRedisAddr, + "stream", cfg.auditRedisStream, + "db", cfg.auditRedisDB, + "tls-enabled", cfg.auditRedisTLS) + } + auditHandler, err := webhookhandler.NewAuditHandler(webhookhandler.AuditHandlerConfig{ DumpDir: cfg.auditDumpPath, MaxRequestBodyBytes: cfg.auditMaxRequestBodyBytes, + Queue: auditQueue, }) fatalIfErr(err, "unable to create audit handler") @@ -290,6 +311,14 @@ type appConfig struct { auditReadTimeout time.Duration auditWriteTimeout time.Duration auditIdleTimeout time.Duration + auditRedisEnabled bool + auditRedisAddr string + auditRedisUsername string + auditRedisPassword string + auditRedisDB int + auditRedisStream string + auditRedisMaxLen int64 + auditRedisTLS bool zapOpts zap.Options } @@ -340,6 +369,22 @@ func parseFlagsWithArgs(fs *flag.FlagSet, args []string) (appConfig, error) { "Write timeout for the dedicated audit ingress HTTPS server.") fs.DurationVar(&cfg.auditIdleTimeout, "audit-idle-timeout", defaultAuditIdleTimeout, "Idle timeout for the dedicated audit ingress HTTPS server.") + fs.BoolVar(&cfg.auditRedisEnabled, "audit-redis-enabled", false, + "If set, accepted audit events are enqueued to Redis Streams for durable buffering.") + fs.StringVar(&cfg.auditRedisAddr, "audit-redis-addr", "", + "Redis server address (:) for audit event queueing.") + fs.StringVar(&cfg.auditRedisUsername, "audit-redis-username", "", + "Optional Redis username for audit event queueing.") + fs.StringVar(&cfg.auditRedisPassword, "audit-redis-password", "", + "Optional Redis password for audit event queueing.") + fs.IntVar(&cfg.auditRedisDB, "audit-redis-db", 0, + "Redis database index for audit event queueing.") + fs.StringVar(&cfg.auditRedisStream, "audit-redis-stream", queue.DefaultRedisAuditStream, + "Redis stream name for audit event queueing.") + fs.Int64Var(&cfg.auditRedisMaxLen, "audit-redis-max-len", 0, + "Approximate max stream length (0 disables trimming).") + fs.BoolVar(&cfg.auditRedisTLS, "audit-redis-tls", false, + "If set, Redis connection for audit queueing uses TLS.") cfg.zapOpts = zap.Options{ // Production mode defaults to JSON encoding, which is easier for log processors to parse. Development: false, @@ -400,6 +445,15 @@ func validateAuditConfig(cfg appConfig) error { if cfg.auditIdleTimeout <= 0 { return fmt.Errorf("audit-idle-timeout must be > 0, got %s", cfg.auditIdleTimeout) } + if cfg.auditRedisEnabled && strings.TrimSpace(cfg.auditRedisAddr) == "" { + return errors.New("audit-redis-addr is required when audit-redis-enabled is true") + } + if cfg.auditRedisDB < 0 { + return fmt.Errorf("audit-redis-db must be >= 0, got %d", cfg.auditRedisDB) + } + if cfg.auditRedisMaxLen < 0 { + return fmt.Errorf("audit-redis-max-len must be >= 0, got %d", cfg.auditRedisMaxLen) + } return nil } diff --git a/cmd/main_audit_server_test.go b/cmd/main_audit_server_test.go index c81b36d6f..05a7add02 100644 --- a/cmd/main_audit_server_test.go +++ b/cmd/main_audit_server_test.go @@ -42,6 +42,11 @@ func TestParseFlagsWithArgs_Defaults(t *testing.T) { assert.Equal(t, 15*time.Second, cfg.auditReadTimeout) assert.Equal(t, 30*time.Second, cfg.auditWriteTimeout) assert.Equal(t, 60*time.Second, cfg.auditIdleTimeout) + assert.False(t, cfg.auditRedisEnabled) + assert.Empty(t, cfg.auditRedisAddr) + assert.Equal(t, "gitopsreverser.audit.events.v1", cfg.auditRedisStream) + assert.Equal(t, int64(0), cfg.auditRedisMaxLen) + assert.False(t, cfg.auditRedisTLS) assert.False(t, cfg.zapOpts.Development) } @@ -69,6 +74,14 @@ func TestParseFlagsWithArgs_CustomAuditValues(t *testing.T) { "--audit-read-timeout=5s", "--audit-write-timeout=8s", "--audit-idle-timeout=13s", + "--audit-redis-enabled", + "--audit-redis-addr=127.0.0.1:6379", + "--audit-redis-username=user", + "--audit-redis-password=pass", + "--audit-redis-db=2", + "--audit-redis-stream=gitopsreverser.audit.custom", + "--audit-redis-max-len=1000", + "--audit-redis-tls", } cfg, err := parseFlagsWithArgs(fs, args) @@ -83,6 +96,14 @@ func TestParseFlagsWithArgs_CustomAuditValues(t *testing.T) { assert.Equal(t, 5*time.Second, cfg.auditReadTimeout) assert.Equal(t, 8*time.Second, cfg.auditWriteTimeout) assert.Equal(t, 13*time.Second, cfg.auditIdleTimeout) + assert.True(t, cfg.auditRedisEnabled) + assert.Equal(t, "127.0.0.1:6379", cfg.auditRedisAddr) + assert.Equal(t, "user", cfg.auditRedisUsername) + assert.Equal(t, "pass", cfg.auditRedisPassword) + assert.Equal(t, 2, cfg.auditRedisDB) + assert.Equal(t, "gitopsreverser.audit.custom", cfg.auditRedisStream) + assert.Equal(t, int64(1000), cfg.auditRedisMaxLen) + assert.True(t, cfg.auditRedisTLS) } func TestParseFlagsWithArgs_FallsBackToWebhookCertPath(t *testing.T) { @@ -113,6 +134,18 @@ func TestParseFlagsWithArgs_InvalidAuditSettings(t *testing.T) { name: "invalid read timeout", args: []string{"--audit-read-timeout=0s"}, }, + { + name: "redis enabled without address", + args: []string{"--audit-redis-enabled"}, + }, + { + name: "invalid redis db", + args: []string{"--audit-redis-db=-1"}, + }, + { + name: "invalid redis max len", + args: []string{"--audit-redis-max-len=-1"}, + }, } for _, tt := range tests { diff --git a/config/deployment.yaml b/config/deployment.yaml index 308c32a6b..84dd58ae7 100644 --- a/config/deployment.yaml +++ b/config/deployment.yaml @@ -29,6 +29,9 @@ spec: - --health-probe-bind-address=:8081 - --webhook-cert-path=/tmp/k8s-admission-server/admission-server-certs - --audit-cert-path=/tmp/k8s-audit-server/audit-server-certs + - --audit-redis-enabled + - --audit-redis-addr=valkey.valkey-e2e.svc.cluster.local:6379 + - --audit-redis-stream=gitopsreverser.audit.events.v1 command: - /manager env: diff --git a/docs/design/audit-webhook-redis-queue-plan.md b/docs/design/audit-webhook-redis-queue-plan.md new file mode 100644 index 000000000..f813ec34a --- /dev/null +++ b/docs/design/audit-webhook-redis-queue-plan.md @@ -0,0 +1,181 @@ +# Audit Webhook Only + Durable Redis Queue Plan + +## Goal + +Move GitOps Reverser to a single ingestion path: + +1. Kubernetes Audit Webhook ingress only +2. Durable Redis-backed queue for all incoming audit events +3. Replayable, at-least-once delivery into Git processing + +Note: this document uses **Redis** terminology for the queue contract. Your new Valkey instance is treated as a +Redis-compatible runtime. + +## Why This Is the Next Step + +Current docs and code show: + +1. Audit ingress is already wired (`/audit-webhook/{clusterID}`) and stable. +2. Current queueing to Git workers is in-process/in-memory (`BranchWorker` channel), which is not durable. +3. Existing design direction already points to a durable queue layer and HA worker model. + +This plan closes the gap by making audit ingress durable first, then cutting over routing and scaling behavior. + +## Architecture Target + +```text +Kube API Audit Backend + -> /audit-webhook/{clusterID} + -> Audit Handler (validate + normalize) + -> Redis Stream (durable enqueue) + -> Redis Consumer Group worker(s) + -> EventRouter / GitTargetEventStream / BranchWorker + -> Git commit/push +``` + +## Data Contract + +Use one Redis stream initially: + +1. Stream: `gitopsreverser.audit.events.v1` +2. Consumer group: `gitopsreverser-writers` +3. Event fields: + - `event_id` (deterministic idempotency key) + - `audit_id` + - `cluster_id` + - `verb` + - `api_version` + - `resource` + - `namespace` + - `name` + - `user` + - `stage_timestamp` + - `payload_json` (normalized event for downstream mapping) + +## Phased Plan + +### Phase 0: Feature-flagged foundation + +1. Add `internal/queue/` package with interfaces: + - `Producer.Enqueue(ctx, event) error` + - `Consumer.Run(ctx, handler) error` +2. Add queue mode config: + - `queue.mode=inmemory|redis` (default `inmemory` for safe rollout) + - `redis.addr`, `redis.username`, `redis.password`, `redis.tls`, `redis.stream`, `redis.group` +3. Add Helm values under `queue` and `redis` while keeping current behavior unchanged by default. + +### Phase 1: Audit ingress -> Redis durable enqueue + +1. Extend `internal/webhook/audit_handler.go` to publish every accepted audit event to Redis. +2. Keep current file dump behavior as optional debug path. +3. Return `5xx` on enqueue failure so Kubernetes audit backend retries. +4. Preserve request body size guardrails and path contract. + +Exit criteria: + +1. All accepted audit events are visible in stream (`XADD` count increases). +2. Handler correctness tests pass for success/failure paths. + +### Phase 2: Redis consumer -> existing processing pipeline + +1. Implement consumer worker runnable in `cmd/main.go` lifecycle. +2. Consume via consumer group (`XREADGROUP`) and `XACK` only after successful handoff to existing event pipeline. +3. On worker crash/stall, reclaim stuck entries via `XAUTOCLAIM`. +4. Start with one consumer process, then allow horizontal scaling. + +Exit criteria: + +1. No dropped events during pod restart test. +2. Pending Entries List (PEL) returns to steady state after failures. + +### Phase 3: Idempotency and ordering controls + +1. Add dedupe key in Redis: + - key: `gitopsreverser:dedupe:` + - set with TTL (for example 24h) before downstream processing. +2. Partition lock per `{provider,branch}` using `SET NX EX` lock keys to avoid concurrent writers. +3. Define lock TTL renewal while worker is active. + +Exit criteria: + +1. Duplicate deliveries do not produce duplicate Git writes. +2. Parallel workers do not race on same branch writes. + +### Phase 4: Audit-only ingestion cutover + +1. Add runtime switch to disable validating-webhook correlation path. +2. Keep watch/reconcile pieces required for seed and rule-change reconciliation. +3. Remove dual-path assumptions from docs and operator config defaults. + +Exit criteria: + +1. Ingestion path for ongoing events is audit-only in production profile. +2. Reconcile startup/rule-change still works. + +### Phase 5: Performance and durability validation + +1. Add metrics: + - `audit_ingest_requests_total` + - `audit_enqueue_latency_seconds` + - `redis_stream_lag` + - `redis_pel_entries` + - `event_end_to_end_latency_seconds` + - `event_dropped_total` +2. Run repeatable load test profiles (low, nominal, burst). +3. Compare baseline (in-memory path) vs Redis mode on: + - p50/p95/p99 ingest latency + - sustained events/sec + - recovery time after Git outage or pod restart + - memory footprint under burst load + +Exit criteria: + +1. Throughput and reliability targets are met or tradeoff is explicitly accepted. +2. Performance report is committed under `docs/ci/`. + +## Performance Expectations + +Likely impact: + +1. Slightly higher per-request latency due to networked enqueue. +2. Significantly better durability and restart recovery. +3. Better burst handling when Git is slow/unavailable. + +Practical expectation is improved reliability under failure and load spikes, with small steady-state latency overhead. + +## Testing Strategy + +1. Unit tests: + - Redis producer/consumer behavior + - enqueue failure handling in audit handler + - idempotency and lock behavior +2. Integration tests: + - consumer group ack/reclaim flows + - restart recovery with pending messages +3. E2E: + - send audit events, verify Git output and Redis ack behavior + - chaos case: stop worker, produce events, restart worker, verify replay + +## Rollout Strategy + +1. Stage A: `queue.mode=inmemory` in production, `redis` in non-prod shadow. +2. Stage B: enable `redis` for one environment with one replica. +3. Stage C: scale replicas and validate lock/ordering behavior. +4. Stage D: set audit-only ingestion as default. + +## Open Decisions + +1. One global stream vs stream-per-cluster/per-target +2. Dedupe TTL duration +3. Exact partition key for writer locks (`repo+branch` vs `target+branch`) +4. Dead-letter policy for poison events + +## First Implementation Slice (Recommended) + +Deliver first: + +1. Redis producer in audit handler (Phase 1) +2. Minimal consumer runnable with `XREADGROUP` + `XACK` (Phase 2) +3. Basic lag/PEL metrics (part of Phase 5 instrumentation) + +This gives durable ingress quickly and lets you measure performance early before deeper refactors. diff --git a/go.mod b/go.mod index ea44b2244..1107015c7 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.3.0 // indirect + github.com/alicebob/miniredis/v2 v2.37.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect @@ -42,6 +43,7 @@ require ( github.com/cloudflare/circl v1.6.3 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect @@ -87,12 +89,14 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.19.2 // indirect + github.com/redis/go-redis/v9 v9.18.0 // indirect github.com/sergi/go-diff v1.4.0 // indirect github.com/skeema/knownhosts v1.3.2 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect @@ -100,6 +104,7 @@ require ( go.opentelemetry.io/otel/sdk v1.42.0 // indirect go.opentelemetry.io/otel/trace v1.42.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect diff --git a/go.sum b/go.sum index 338a9db95..400cee36b 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= +github.com/alicebob/miniredis/v2 v2.37.0 h1:RheObYW32G1aiJIj81XVt78ZHJpHonHLHW7OLIshq68= +github.com/alicebob/miniredis/v2 v2.37.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= @@ -37,6 +39,8 @@ github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= @@ -196,6 +200,8 @@ github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEo github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= +github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -229,6 +235,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= @@ -251,6 +259,8 @@ go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4Len go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= diff --git a/internal/queue/redis_audit_queue.go b/internal/queue/redis_audit_queue.go new file mode 100644 index 000000000..6e0c05f24 --- /dev/null +++ b/internal/queue/redis_audit_queue.go @@ -0,0 +1,170 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/redis/go-redis/v9" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" +) + +const ( + // DefaultRedisAuditStream is the default stream used for audit ingestion events. + DefaultRedisAuditStream = "gitopsreverser.audit.events.v1" +) + +// RedisAuditQueueConfig configures a Redis-backed audit queue. +type RedisAuditQueueConfig struct { + Addr string + Username string + AuthValue string + DB int + Stream string + MaxLen int64 + TLSEnabled bool +} + +// RedisAuditQueue enqueues audit events into a Redis stream. +type RedisAuditQueue struct { + client *redis.Client + stream string + maxLen int64 +} + +// NewRedisAuditQueue creates a Redis stream-backed audit queue. +func NewRedisAuditQueue(cfg RedisAuditQueueConfig) (*RedisAuditQueue, error) { + if strings.TrimSpace(cfg.Addr) == "" { + return nil, errors.New("redis address is required") + } + + stream := strings.TrimSpace(cfg.Stream) + if stream == "" { + stream = DefaultRedisAuditStream + } + + options := &redis.Options{ + Addr: cfg.Addr, + Username: cfg.Username, + Password: cfg.AuthValue, + DB: cfg.DB, + } + + if cfg.TLSEnabled { + options.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12} + } + + return &RedisAuditQueue{ + client: redis.NewClient(options), + stream: stream, + maxLen: cfg.MaxLen, + }, nil +} + +// Enqueue writes one audit event to Redis stream storage. +func (q *RedisAuditQueue) Enqueue(ctx context.Context, clusterID string, event auditv1.Event) error { + payload, err := json.Marshal(event) + if err != nil { + return fmt.Errorf("failed to marshal audit event payload: %w", err) + } + + objectRef := event.ObjectRef + apiVersion := "" + resource := "" + namespace := "" + name := "" + if objectRef != nil { + apiVersion = objectRef.APIVersion + resource = objectRef.Resource + namespace = objectRef.Namespace + name = objectRef.Name + } + + values := map[string]any{ + "event_id": buildEventID(clusterID, event), + "audit_id": string(event.AuditID), + "cluster_id": clusterID, + "verb": event.Verb, + "api_version": apiVersion, + "resource": resource, + "namespace": namespace, + "name": name, + "user": event.User.Username, + "stage_timestamp": formatStageTimestamp(event.StageTimestamp.Time), + "payload_json": string(payload), + } + + args := &redis.XAddArgs{ + Stream: q.stream, + ID: "*", + Values: values, + } + if q.maxLen > 0 { + args.MaxLen = q.maxLen + args.Approx = true + } + + if _, err := q.client.XAdd(ctx, args).Result(); err != nil { + return fmt.Errorf("failed to append audit event to redis stream %q: %w", q.stream, err) + } + + return nil +} + +func buildEventID(clusterID string, event auditv1.Event) string { + var builder strings.Builder + builder.WriteString(strings.TrimSpace(clusterID)) + builder.WriteString("|") + builder.WriteString(string(event.AuditID)) + builder.WriteString("|") + builder.WriteString(string(event.Stage)) + builder.WriteString("|") + builder.WriteString(event.Verb) + builder.WriteString("|") + builder.WriteString(event.RequestURI) + builder.WriteString("|") + builder.WriteString(event.StageTimestamp.Time.UTC().Format(time.RFC3339Nano)) + if event.ObjectRef != nil { + builder.WriteString("|") + builder.WriteString(event.ObjectRef.APIVersion) + builder.WriteString("|") + builder.WriteString(event.ObjectRef.Resource) + builder.WriteString("|") + builder.WriteString(event.ObjectRef.Namespace) + builder.WriteString("|") + builder.WriteString(event.ObjectRef.Name) + } + sum := sha256.Sum256([]byte(builder.String())) + return hex.EncodeToString(sum[:]) +} + +func formatStageTimestamp(t time.Time) string { + if t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339Nano) +} diff --git a/internal/queue/redis_audit_queue_test.go b/internal/queue/redis_audit_queue_test.go new file mode 100644 index 000000000..c8302160f --- /dev/null +++ b/internal/queue/redis_audit_queue_test.go @@ -0,0 +1,85 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package queue + +import ( + "context" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + auditv1 "k8s.io/apiserver/pkg/apis/audit/v1" +) + +func TestNewRedisAuditQueue_RequiresAddress(t *testing.T) { + _, err := NewRedisAuditQueue(RedisAuditQueueConfig{}) + require.Error(t, err) +} + +func TestRedisAuditQueue_Enqueue(t *testing.T) { + mr := miniredis.RunT(t) + + queue, err := NewRedisAuditQueue(RedisAuditQueueConfig{ + Addr: mr.Addr(), + Stream: "audit.events.test", + MaxLen: 100, + }) + require.NoError(t, err) + + event := auditv1.Event{ + AuditID: "audit-123", + Verb: "create", + RequestURI: "/api/v1/namespaces/default/configmaps/cm-a", + Stage: "ResponseComplete", + StageTimestamp: metav1.MicroTime{ + Time: time.Date(2026, 3, 5, 10, 0, 0, 0, time.UTC), + }, + ObjectRef: &auditv1.ObjectReference{ + APIVersion: "v1", + Resource: "configmaps", + Namespace: "default", + Name: "cm-a", + }, + } + event.User.Username = "test-user" + + err = queue.Enqueue(context.Background(), "cluster-a", event) + require.NoError(t, err) + + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + entries, err := client.XRange(context.Background(), "audit.events.test", "-", "+").Result() + require.NoError(t, err) + require.Len(t, entries, 1) + + entry := entries[0].Values + assert.Equal(t, "audit-123", entry["audit_id"]) + assert.Equal(t, "cluster-a", entry["cluster_id"]) + assert.Equal(t, "create", entry["verb"]) + assert.Equal(t, "v1", entry["api_version"]) + assert.Equal(t, "configmaps", entry["resource"]) + assert.Equal(t, "default", entry["namespace"]) + assert.Equal(t, "cm-a", entry["name"]) + assert.Equal(t, "test-user", entry["user"]) + assert.NotEmpty(t, entry["event_id"]) + assert.NotEmpty(t, entry["payload_json"]) +} diff --git a/internal/webhook/audit_handler.go b/internal/webhook/audit_handler.go index cdedd158a..fe8659f83 100644 --- a/internal/webhook/audit_handler.go +++ b/internal/webhook/audit_handler.go @@ -61,6 +61,14 @@ type AuditHandlerConfig struct { DumpDir string // MaxRequestBodyBytes is the maximum accepted HTTP request body size. MaxRequestBodyBytes int64 + // Queue enqueues accepted audit events to a durable backend. + // If nil, queueing is disabled. + Queue AuditEventQueue +} + +// AuditEventQueue persists accepted audit events for downstream processing. +type AuditEventQueue interface { + Enqueue(ctx context.Context, clusterID string, event auditv1.Event) error } // AuditHandler handles incoming audit events and collects telemetry. @@ -185,6 +193,12 @@ func (h *AuditHandler) processEvents(ctx context.Context, clusterID string, even return fmt.Errorf("failed to check audit event: %w", err) } + if h.config.Queue != nil { + if err := h.config.Queue.Enqueue(ctx, clusterID, auditEventV1); err != nil { + return fmt.Errorf("failed to enqueue audit event %q: %w", auditEvent.AuditID, err) + } + } + gvr := h.extractGVR(&auditEvent) action := auditEvent.Verb diff --git a/internal/webhook/audit_handler_test.go b/internal/webhook/audit_handler_test.go index f3e6a7094..5981dea79 100644 --- a/internal/webhook/audit_handler_test.go +++ b/internal/webhook/audit_handler_test.go @@ -22,10 +22,12 @@ import ( "bytes" "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "os" "strings" + "sync" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,6 +41,26 @@ import ( audit "k8s.io/apiserver/pkg/apis/audit" ) +type fakeAuditEventQueue struct { + mu sync.Mutex + events []auditv1.Event + err error + calls int + clusters []string +} + +func (q *fakeAuditEventQueue) Enqueue(_ context.Context, clusterID string, event auditv1.Event) error { + q.mu.Lock() + defer q.mu.Unlock() + q.calls++ + q.clusters = append(q.clusters, clusterID) + q.events = append(q.events, event) + if q.err != nil { + return q.err + } + return nil +} + func TestMain(m *testing.M) { // Initialize metrics for tests _, err := telemetry.InitOTLPExporter(context.Background()) @@ -419,6 +441,44 @@ func TestAuditHandler_RejectsOversizedBody(t *testing.T) { assert.Contains(t, w.Body.String(), "request body too large") } +func TestAuditHandler_EnqueuesAcceptedEvents(t *testing.T) { + queue := &fakeAuditEventQueue{} + handler, err := NewAuditHandler(AuditHandlerConfig{ + Queue: queue, + }) + require.NoError(t, err) + + body := `{"kind":"EventList","apiVersion":"audit.k8s.io/v1","items":[{"kind":"Event","auditID":"queued-1","verb":"update","stage":"ResponseComplete","user":{"username":"test-user"},"objectRef":{"resource":"configmaps","namespace":"default","name":"cm-a","apiVersion":"v1"}}]}` + req := httptest.NewRequest(http.MethodPost, "/audit-webhook/cluster-a", bytes.NewReader([]byte(body))) + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + require.Equal(t, 1, queue.calls) + require.Len(t, queue.events, 1) + assert.Equal(t, "cluster-a", queue.clusters[0]) + assert.Equal(t, "queued-1", string(queue.events[0].AuditID)) +} + +func TestAuditHandler_EnqueueFailureReturnsInternalServerError(t *testing.T) { + queue := &fakeAuditEventQueue{err: errors.New("queue down")} + handler, err := NewAuditHandler(AuditHandlerConfig{ + Queue: queue, + }) + require.NoError(t, err) + + body := `{"kind":"EventList","apiVersion":"audit.k8s.io/v1","items":[{"kind":"Event","auditID":"queued-1","verb":"create","stage":"ResponseComplete","user":{"username":"test-user"},"objectRef":{"resource":"secrets","namespace":"default","name":"secret-a","apiVersion":"v1"}}]}` + req := httptest.NewRequest(http.MethodPost, "/audit-webhook/cluster-a", bytes.NewReader([]byte(body))) + w := httptest.NewRecorder() + + handler.ServeHTTP(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Code) + assert.Contains(t, w.Body.String(), "failed to enqueue audit event") + require.Equal(t, 1, queue.calls) +} + func TestExtractClusterID(t *testing.T) { tests := []struct { name string diff --git a/test/e2e/audit_redis_e2e_test.go b/test/e2e/audit_redis_e2e_test.go new file mode 100644 index 000000000..721e04fb1 --- /dev/null +++ b/test/e2e/audit_redis_e2e_test.go @@ -0,0 +1,121 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright 2025 ConfigButler + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/redis/go-redis/v9" +) + +const ( + defaultAuditRedisStream = "gitopsreverser.audit.events.v1" +) + +var _ = Describe("Audit Redis Queue", Label("audit-redis"), Ordered, func() { + It("should enqueue incoming audit webhook events into a Redis stream", func() { + streamName := defaultAuditRedisStream + testConfigMapName := fmt.Sprintf("audit-redis-test-cm-%d", GinkgoRandomSeed()) + + By("connecting to Valkey through the e2e port-forward") + client := redis.NewClient(&redis.Options{ + Addr: valkeyPortForwardAddr(), + }) + Eventually(func(g Gomega) { + g.Expect(client.Ping(context.Background()).Err()).NotTo(HaveOccurred()) + }, 30*time.Second, 2*time.Second).Should(Succeed()) + + By("clearing the dedicated stream for deterministic assertions") + _, err := client.Del(context.Background(), streamName).Result() + Expect(err).NotTo(HaveOccurred()) + + By("triggering kube-apiserver audit events with a ConfigMap create/update") + _, err = kubectlRunInNamespace( + namespace, + "delete", + "configmap", + testConfigMapName, + "--ignore-not-found=true", + ) + Expect(err).NotTo(HaveOccurred()) + + _, err = kubectlRunInNamespace( + namespace, + "create", + "configmap", + testConfigMapName, + "--from-literal=test=audit-redis", + ) + Expect(err).NotTo(HaveOccurred()) + + _, err = kubectlRunInNamespace( + namespace, + "patch", + "configmap", + testConfigMapName, + "--type=merge", + "--patch", + `{"data":{"test":"audit-redis-updated"}}`, + ) + Expect(err).NotTo(HaveOccurred()) + + By("verifying Redis stream receives an entry for the ConfigMap audit event") + Eventually(func(g Gomega) { + entries, readErr := client.XRange(context.Background(), streamName, "-", "+").Result() + g.Expect(readErr).NotTo(HaveOccurred()) + g.Expect(entries).NotTo(BeEmpty()) + + found := false + for _, entry := range entries { + resource, _ := entry.Values["resource"].(string) + name, _ := entry.Values["name"].(string) + clusterID, _ := entry.Values["cluster_id"].(string) + payload, _ := entry.Values["payload_json"].(string) + if resource == "configmaps" && name == testConfigMapName && clusterID == "kind-e2e" && payload != "" { + found = true + break + } + } + g.Expect(found).To(BeTrue(), "expected ConfigMap audit event to be enqueued in Redis stream") + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + By("cleaning up test ConfigMap") + _, _ = kubectlRunInNamespace( + namespace, + "delete", + "configmap", + testConfigMapName, + "--ignore-not-found=true", + ) + }) +}) + +func valkeyPortForwardAddr() string { + port := strings.TrimSpace(os.Getenv("VALKEY_PORT")) + if port == "" { + port = "16379" + } + return "127.0.0.1:" + port +} From cbe662bb99ec61f5aced064b0fefe376bd764d26 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 13 Mar 2026 08:06:24 +0000 Subject: [PATCH 2/8] chore: Some docs, let's see if the tests are still failing --- README.md | 1 - docs/KUBECON_VOTING_ROLLOUT_PLAN.md | 302 ++++++++++++++++++ docs/KUBERNETES_API_DISCOVERY_AND_WATCHING.md | 102 ++++++ 3 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 docs/KUBECON_VOTING_ROLLOUT_PLAN.md create mode 100644 docs/KUBERNETES_API_DISCOVERY_AND_WATCHING.md diff --git a/README.md b/README.md index 3de31b485..18c9b93d9 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,6 @@ kubectl create configmap test-config --from-literal=key=value -n default For cluster-wide resources (nodes, CRDs, etc.) or watching multiple namespaces, use [`ClusterWatchRule`](config/samples/clusterwatchrule.yaml). More examples in [`config/samples/`](config/samples/). - ## Usage guidance Avoid infinite loops: Do not point GitOps (Argo CD/Flux) and GitOps Reverser at the same resources in fully automated mode. Recommended patterns: diff --git a/docs/KUBECON_VOTING_ROLLOUT_PLAN.md b/docs/KUBECON_VOTING_ROLLOUT_PLAN.md new file mode 100644 index 000000000..a839895e2 --- /dev/null +++ b/docs/KUBECON_VOTING_ROLLOUT_PLAN.md @@ -0,0 +1,302 @@ +# KubeCon Voting Portal Rollout Plan + +## Goal + +Run a simple audience voting portal on `voting.reversegitops.dev`, hosted from this devcontainer-driven setup, in a way that: + +- is realistic enough for the talk, +- exercises the "path back" from live changes into Git, +- does not introduce avoidable demo-day risk, +- leaves behind useful test coverage for the project. + +## Short Recommendation + +Using this environment is reasonable for a talk demo, but only if we treat it as a staged rollout and not as a last-minute live experiment. + +The biggest risk is not Kubernetes itself. It is the full chain: + +1. audience phones, +2. public DNS, +3. Cloudflare tunnel, +4. devcontainer host, +5. k3d cluster, +6. app behavior under bursty writes, +7. GitOps Reverser behavior under concurrent changes. + +That means we should prove the chain in layers, in this order, and keep the voting app intentionally boring. + +## Proposed Order + +### Phase 1: Define the demo contract + +Before installing anything else, lock down the shape of the demo. + +Deliverables: + +- decide whether votes are anonymous or tied to a session/user identifier, +- decide whether duplicate voting is allowed, +- decide whether results must update live or can refresh every few seconds, +- decide whether GitOps Reverser is in the critical request path or only observing, +- decide what failure is acceptable during the talk. + +Recommendation: + +- keep GitOps Reverser out of the synchronous request path, +- store votes in a simple in-cluster service first, +- let GitOps Reverser observe resulting Kubernetes changes or supporting resources rather than acting as the database. + +Why: + +The talk demo should prove the concept, not depend on every moving piece succeeding within one HTTP request. + +### Phase 2: Create a dedicated talk environment branch + +Introduce a dedicated branch for talk-specific manifests and automation. + +Suggested shape: + +- `main`: normal project development, +- `talk/kubecon-voting`: talk environment manifests and overlays, +- optionally `talk/kubecon-voting-rehearsal`: dry-run branch for repeated practice. + +Recommendation: + +- yes, adding Flux is probably worth it here, +- use Flux to install only the talk-specific layer, +- do not point Flux at the same exact resources GitOps Reverser is writing back for the demo. + +Why: + +- Flux makes it much easier to manage a branch with extra components like the voting app and `cloudflared`, +- it also mirrors the real-world "Git in control" story more clearly, +- but the repo already warns about feedback loops, so we should keep ownership boundaries explicit. + +Suggested ownership split: + +- Flux manages infra and app deployment manifests, +- the voting app handles vote persistence, +- GitOps Reverser observes selected resources or a dedicated demo namespace, +- GitOps Reverser does not write back into the same Flux-managed path that is being continuously reconciled. + +### Phase 3: Make ingress boring and rehearseable + +Get public access working early. + +Recommendation: + +- use `cloudflared` only after the app works locally inside k3d, +- expose one hostname only: `voting.reversegitops.dev`, +- avoid adding extra public endpoints during the talk setup, +- prepare a local fallback URL or port-forward for rehearsal. + +What to validate: + +- DNS resolution, +- tunnel reconnect behavior, +- TLS from a real phone on real mobile data, +- behavior after restarting `cloudflared`, +- behavior after restarting the app deployment, +- behavior after a devcontainer restart, if that is realistic for your host setup. + +### Phase 3.5: Add a private TTY path for live slides + +In addition to the public voting endpoint, keep a separate private path into the dev environment for the talk itself. + +Recommendation: + +- run a TTY server in the background on the host or inside the dev environment, +- do not expose that TTY service to the public internet, +- open only the required port on the private side, +- access it through your existing VPN from the slide deck or demo machine, +- treat this as the preferred operator/demo access path during the talk. + +Why: + +- you have already been testing this workflow, +- it separates audience traffic from presenter access, +- it avoids turning shell access into a public-facing risk, +- it is likely the safest and most reliable option for live terminal control. + +What to validate: + +- VPN reachability from the presentation machine, +- TTY reconnect behavior after network hiccups, +- terminal latency while the voting demo is under load, +- behavior after restarting the TTY server, +- whether the service survives a devcontainer restart or can be brought back quickly. + +### Phase 4: Add a minimal end-to-end app smoke test + +Yes, some of this belongs in e2e, but only the deterministic parts. + +Recommended e2e scope: + +- deploy the voting app into the existing k3d harness, +- submit a small number of votes, +- verify the expected Kubernetes-side effect exists, +- verify GitOps Reverser still behaves correctly if that side effect is something it watches, +- verify the app remains usable after a rollout/restart. + +This should be a normal e2e or quickstart-style smoke test, not a high-load test. + +Why not put the full load test into the normal e2e suite: + +- CI-friendly e2e should stay deterministic and reasonably fast, +- "100 people hammering phones at once" is more of a scenario test than a correctness test, +- load tests often fail for environmental reasons and become noisy in CI. + +### Phase 5: Add a dedicated load test target that reuses the e2e harness + +This is where I would put the audience simulation. + +Recommendation: + +- reuse the existing k3d/Gitea/e2e setup, +- add a separate load-test target rather than folding it into `make test-e2e`, +- keep one tiny load-related assertion in e2e if we want regression coverage, +- keep the real stress scenario as an opt-in target for rehearsal. + +Suggested shape: + +- `make test-load-voting` or `make test-e2e-voting-load` +- deploy the talk stack, +- generate 100-200 virtual users with bursty vote submission, +- capture: + - request latency, + - error rate, + - app pod CPU/memory, + - tunnel stability, + - API server pressure, + - GitOps Reverser queue depth / processing latency / commit rate, + - Git write behavior under bursts. + +Success criteria: + +- no user-visible failures for the expected audience size, +- no uncontrolled backlog growth, +- no Git corruption or push thrashing, +- recovery after the burst finishes, +- acceptable latency on cheap mobile devices and conference Wi-Fi. + +My recommendation on tooling: + +- use a simple HTTP load generator outside the normal Go e2e assertions, +- keep result evaluation threshold-based, +- archive a markdown summary or artifact from rehearsal runs. + +### Phase 6: Observe the full GitOps story under rehearsal conditions + +Once the app and load path are stable, add the "interesting" GitOps proof. + +What to prove: + +- votes or vote-driven actions create a clean, understandable Git history, +- the generated YAML is meaningful enough to show on stage, +- the commit frequency under burst load stays sane, +- Flux and GitOps Reverser do not fight each other, +- rollback and redeploy behavior are understandable. + +This is the place to decide whether every vote should produce a Git-visible change. + +Recommendation: + +- probably not. + +For the talk, a better pattern may be: + +- app records votes normally, +- a summarizer periodically writes an aggregate ConfigMap or CR, +- GitOps Reverser captures the aggregate state, +- the Git history becomes legible instead of becoming 100 near-identical commits. + +That will tell a cleaner story on stage and reduce risk significantly. + +### Phase 7: Rehearsal and failure drills + +Before KubeCon, run at least two full rehearsals with the public hostname. + +Rehearsal checklist: + +- cold start from the current branch, +- Flux sync completes, +- tunnel comes up, +- real phone can vote, +- 100-user synthetic load passes, +- GitOps Reverser output is still clean, +- restart one pod during voting, +- restart `cloudflared`, +- confirm recovery time is acceptable, +- verify logs and dashboards are easy to read during stress. + +## What I Would Build First + +If we want the safest path, I would build in this order: + +1. a very small voting app with one namespace and one hostname, +2. local k3d deployment without public ingress, +3. Cloudflare tunnel exposure, +4. Flux-managed talk branch, +5. smoke e2e coverage, +6. separate load-test target, +7. GitOps Reverser integration for the stage-worthy story, +8. rehearsal automation and dashboards. + +## What I Would Not Do + +- I would not make Git commits part of the synchronous vote request. +- I would not let Flux and GitOps Reverser continuously own the same resources. +- I would not rely on the conference network for the first real test. +- I would not make the live demo depend on per-vote Git writes unless that behavior has already survived repeated load drills. + +## Suggested Repo Changes + +I would expect this work to land as a few separate tracks: + +### 1. Talk environment docs + +- a dedicated setup doc for the voting environment, +- architecture diagram for request flow and Git flow, +- rehearsal checklist, +- rollback/fallback playbook. + +### 2. Deployment assets + +- talk-specific manifests or Flux kustomizations, +- `cloudflared` manifests, +- TTY server background service or startup configuration, +- voting app manifests, +- optional monitoring/dashboard manifests. + +### 3. Test assets + +- e2e smoke for voting flow, +- separate load test harness and Make target, +- metrics capture or summary output from rehearsal runs. + +## Open Questions + +These are the main decisions still worth answering before implementation: + +1. What exactly should GitOps Reverser record during the demo: every vote, periodic aggregates, or only app/config changes? +2. Is the voting app meant to be disposable for the talk, or should it become a reusable example in this repo? +3. Do you want the public service to survive a devcontainer restart, or is a manual recovery step acceptable? +4. Are you comfortable depending on `cloudflared` for the live talk, or do you want a second ingress fallback? +5. Is conference Wi-Fi the expected client path, or should we test explicitly from mobile data as the primary path? +6. Which TTY server do you want to standardize on for the slide integration, and how should it be started automatically? +7. Do you want Flux only for the talk environment, or is this intended to be the first real Flux integration for the repo itself? +8. What is the acceptable audience limit: 50, 100, 200? +9. What is the acceptable degraded mode: slow updates, delayed Git commits, read-only results page, or full failover? + +## Concrete Next Step + +The best next step is to turn this into a small implementation track: + +1. define the demo contract, +2. create the talk branch and manifests skeleton, +3. ship a tiny voting app locally, +4. expose it through `cloudflared`, +5. add one smoke e2e, +6. add one separate load-test target, +7. rehearse against the real hostname. + +If you want, the next pass can turn this document into a more execution-ready checklist with specific repo paths, Make targets, and proposed Flux layout. diff --git a/docs/KUBERNETES_API_DISCOVERY_AND_WATCHING.md b/docs/KUBERNETES_API_DISCOVERY_AND_WATCHING.md new file mode 100644 index 000000000..21b42ac9e --- /dev/null +++ b/docs/KUBERNETES_API_DISCOVERY_AND_WATCHING.md @@ -0,0 +1,102 @@ +# Kubernetes API Discovery and Watching in GitOps Reverser + +This document explains how GitOps Reverser decides what it can watch, and gives Kubernetes API background for +learning. + +## TL;DR + +- A watched resource does **not** need to be a CRD. +- GitOps Reverser can watch any discoverable Kubernetes resource that supports `list` and `watch`, including: + - Built-in resources (for example `configmaps.v1`) + - CRD-backed resources + - Aggregated API resources served through `APIService` +- Practical rule: if the resource appears in API discovery and has `list/watch` verbs, it is a candidate. + +## Kubernetes API Theory (Quick Primer) + +Kubernetes APIs are addressed as **Group / Version / Resource (GVR)**: + +- Group: `""` (core), `apps`, `rbac.authorization.k8s.io`, `metrics.k8s.io`, etc. +- Version: `v1`, `v1beta1`, etc. +- Resource: plural name such as `pods`, `deployments`, `customresources`. + +Examples: + +- Core API: `"" / v1 / configmaps` +- Apps API: `apps / v1 / deployments` +- Custom API: `shop.example.com / v1 / icecreamorders` + +### Where resources come from + +Kubernetes can serve resources from multiple backends: + +- Built-in APIs (compiled into kube-apiserver) +- CRDs (registered through `CustomResourceDefinition`) +- Aggregated APIs (registered through `APIService`, then served by an extension API server) + +From a client perspective, all of these are exposed through API discovery endpoints (`/api` and `/apis`). + +## How GitOps Reverser Chooses Watch Targets + +At runtime, the watch manager does this: + +1. Build requested GVRs from active `WatchRule` and `ClusterWatchRule`. +2. Query API discovery (`ServerPreferredResources`). +3. Keep only resources that: + - exist in discovery, + - support both `list` and `watch`, + - match the requested scope (`Namespaced` vs `Cluster`). +4. Start dynamic informers for the remaining GVRs. + +This means the watch decision is based on **discoverability and verbs**, not on "is it a CRD?". + +## CRD vs APIService: What This Means in Practice + +### CRDs + +CRDs are supported when: + +- their GVR is in discovery, +- the resource supports `list` and `watch`, +- the rule points to a concrete GVR that matches scope. + +### APIService-backed resources + +APIService-backed resources are also supported under the same conditions: + +- the `APIService` is healthy and available, +- the resource appears in discovery, +- the API server exposes `list` and `watch` for that resource. + +So yes, you can watch resources served through `APIService`; CRD is not required. + +## Current Product Constraints (Important) + +- Informer planning currently needs concrete GVRs: + - one concrete API group (not `*`) + - one concrete API version (not `*`) + - concrete resource name (not `*`, no subresource forms) +- Wildcard expansion across discovery is not implemented yet. +- A small built-in exclusion list skips noisy resources (for example `pods`, `events`, `leases`, `jobs`). +- RBAC still applies: the operator must be allowed to `get/list/watch` the target resources. +- Scope still applies: + - `WatchRule` is namespace-scoped + - `ClusterWatchRule` can watch cluster-scoped and namespaced resources (by rule scope) + +## Quick Verification Commands + +Use these to check if a resource is likely watchable: + +```bash +# 1) Is the resource discoverable? +kubectl api-resources | grep -E 'icecreamorders|' + +# 2) Check served API group/version directly (example) +kubectl get --raw /apis/metrics.k8s.io/v1beta1 + +# 3) Validate operator permissions +kubectl auth can-i list --all-namespaces +kubectl auth can-i watch --all-namespaces +``` + +If a resource is missing from discovery or does not support `watch`, GitOps Reverser will not start an informer for it. From b39d93f26b37a062aff6b4e7f904a31b8a0d7bc6 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 13 Mar 2026 08:23:46 +0000 Subject: [PATCH 3/8] fix: Really patch the status so that the test makes sense --- test/e2e/e2e_test.go | 21 +++++++++++++-------- test/e2e/templates/icecreamorder-crd.yaml | 2 ++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 029f3af7e..c8972a728 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -1280,8 +1280,15 @@ var _ = Describe("Manager", Ordered, func() { WithPolling(2 * time.Second). Should(Succeed()) + By("verifying the IceCreamOrder still exists before status update") + verifyCRDInstanceExists := func(g Gomega) { + _, err := kubectlRunInNamespace(namespace, "get", "icecreamorder", crdInstanceName) + g.Expect(err).NotTo(HaveOccurred(), "IceCreamOrder should exist before status patch") + } + Eventually(verifyCRDInstanceExists, 15*time.Second, time.Second).Should(Succeed()) + By("applying status update to the IceCreamOrder CR") - statusPatch := `{"status":{"orderStatus":"pending","preparationTime":"5m","totalPrice":"$12.50"}}` + statusPatch := `{"status":{"phase":"Preparing","cost":12.5,"message":"Queued for pickup"}}` statusOutput, statusErr := kubectlRunInNamespace( namespace, "patch", @@ -1292,12 +1299,8 @@ var _ = Describe("Manager", Ordered, func() { "-p", statusPatch, ) - if statusErr != nil { - // Status subresource might not be configured for this CRD, which is fine for this test - By(fmt.Sprintf("⚠️ Status patch not supported (expected): %v", statusErr)) - } else { - By(fmt.Sprintf("✅ Status patched successfully: %s", statusOutput)) - } + Expect(statusErr).NotTo(HaveOccurred(), "Status subresource patch should succeed") + By(fmt.Sprintf("status patched successfully: %s", statusOutput)) By("getting current git commit hash") gitRevCmd := exec.Command("git", "rev-parse", "HEAD") @@ -1330,7 +1333,9 @@ var _ = Describe("Manager", Ordered, func() { g.Expect(readErr).NotTo(HaveOccurred()) g.Expect(string(content)).NotTo(ContainSubstring("status:"), "CRD instance file should still NOT contain status field after status update") - g.Expect(string(content)).NotTo(ContainSubstring("orderStatus"), + g.Expect(string(content)).NotTo(ContainSubstring("phase:"), + "CRD instance file should NOT contain status phase content") + g.Expect(string(content)).NotTo(ContainSubstring("Queued for pickup"), "CRD instance file should NOT contain status content") } Eventually(verifyStatusNotCommitted).Should(Succeed()) diff --git a/test/e2e/templates/icecreamorder-crd.yaml b/test/e2e/templates/icecreamorder-crd.yaml index 9fa1c21e0..4d87345e6 100644 --- a/test/e2e/templates/icecreamorder-crd.yaml +++ b/test/e2e/templates/icecreamorder-crd.yaml @@ -13,6 +13,8 @@ spec: served: true # One and only one version must be marked as the storage version. storage: true + subresources: + status: {} schema: # openAPIV3Schema is the schema for validating custom objects. openAPIV3Schema: From ed441f089a86806d3a645062b7ee538c657c2eaa Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 13 Mar 2026 09:21:02 +0000 Subject: [PATCH 4/8] chore: Get flux onboard to simplify the makefile and make it easier to install other additions in the e2e cluster --- .devcontainer/Dockerfile | 15 +- .devcontainer/README.md | 2 +- .github/workflows/ci.yml | 2 +- Dockerfile | 2 +- Makefile | 99 ++-- docs/ci/CI_NON_ROOT_USER_ANALYSIS.md | 4 +- docs/design/flux-e2e-services-plan.md | 458 ++++++++++++++++++ go.mod | 2 +- .../setup/flux-services/kustomization.yaml | 33 ++ .../namespaces/cert-manager.yaml | 4 + .../flux-services/namespaces/gitea-e2e.yaml | 4 + .../flux-services/namespaces/valkey-e2e.yaml | 4 + .../flux-services/releases/cert-manager.yaml | 28 ++ .../setup/flux-services/releases/gitea.yaml | 27 ++ .../setup/flux-services/releases/valkey.yaml | 27 ++ .../repositories/gitea-charts.yaml | 8 + .../flux-services/repositories/jetstack.yaml | 8 + .../flux-services/repositories/valkey.yaml | 8 + .../flux-services/values}/gitea-values.yaml | 23 +- .../flux-services/values}/valkey-values.yaml | 0 20 files changed, 672 insertions(+), 86 deletions(-) create mode 100644 docs/design/flux-e2e-services-plan.md create mode 100644 test/e2e/setup/flux-services/kustomization.yaml create mode 100644 test/e2e/setup/flux-services/namespaces/cert-manager.yaml create mode 100644 test/e2e/setup/flux-services/namespaces/gitea-e2e.yaml create mode 100644 test/e2e/setup/flux-services/namespaces/valkey-e2e.yaml create mode 100644 test/e2e/setup/flux-services/releases/cert-manager.yaml create mode 100644 test/e2e/setup/flux-services/releases/gitea.yaml create mode 100644 test/e2e/setup/flux-services/releases/valkey.yaml create mode 100644 test/e2e/setup/flux-services/repositories/gitea-charts.yaml create mode 100644 test/e2e/setup/flux-services/repositories/jetstack.yaml create mode 100644 test/e2e/setup/flux-services/repositories/valkey.yaml rename test/e2e/{ => setup/flux-services/values}/gitea-values.yaml (70%) rename test/e2e/{ => setup/flux-services/values}/valkey-values.yaml (100%) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 70dfe8448..87984e3b1 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,6 +1,6 @@ # Multi-stage Dockerfile for GitOps Reverser # Stage 1: CI base image with essential build tools -FROM golang:1.26.0-bookworm AS ci +FROM golang:1.26.1-bookworm AS ci # Avoid warnings by switching to noninteractive ENV DEBIAN_FRONTEND=noninteractive @@ -32,12 +32,18 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # Tool versions - centralized for easy updates +# This devcontainer currently supports amd64 tool downloads only. ENV KUBECTL_VERSION=v1.35.0 \ KUSTOMIZE_VERSION=5.8.1 \ KUBEBUILDER_VERSION=4.11.1 \ GOLANGCI_LINT_VERSION=v2.10.1 \ HELM_VERSION=v4.0.0 \ - K3D_VERSION=v5.8.3 + K3D_VERSION=v5.8.3 \ + FLUX_VERSION=2.8.0 + +# Fail early on unsupported architectures instead of producing a partial image. +RUN test "$(dpkg --print-architecture)" = "amd64" \ + || (echo "This devcontainer currently supports amd64 only." && exit 1) # Install kubectl RUN curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ @@ -67,6 +73,11 @@ RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/i RUN curl -fsSL -o /usr/local/bin/k3d "https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64" \ && chmod +x /usr/local/bin/k3d +# Install Flux CLI +RUN curl -fsSL "https://github.com/fluxcd/flux2/releases/download/v${FLUX_VERSION}/flux_${FLUX_VERSION}_linux_amd64.tar.gz" \ + | tar -xzO flux > /usr/local/bin/flux \ + && chmod +x /usr/local/bin/flux + # Set working directory WORKDIR /workspaces diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 90ac4dc43..5b5f07644 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -12,7 +12,7 @@ Quick-start development environment with all tools pre-installed. ### Verify ```bash -go version # 1.26.0 +go version # 1.26.1 kind version # v0.30.0 kubectl version # v1.32.3 golangci-lint version # v2.4.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f082c62a3..05ae27772 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: branches: [main, develop] env: - GO_VERSION: "1.26.0" + GO_VERSION: "1.26.1" REGISTRY: ghcr.io IMAGE_NAME: configbutler/gitops-reverser IMAGE_TAG: ci-${{ github.sha }} diff --git a/Dockerfile b/Dockerfile index a29fb3316..480813e13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.26.0 AS builder +FROM golang:1.26.1 AS builder # Automatic platform arguments provided by Docker BuildKit ARG TARGETOS diff --git a/Makefile b/Makefile index 8042addaf..a7129170d 100644 --- a/Makefile +++ b/Makefile @@ -269,6 +269,7 @@ dist/install.yaml: $(HELM_SYNC_OUTPUTS) $$(CHART_INPUTS) | dist ## Generate cons KUBECTL ?= kubectl K3D ?= k3d HELM ?= helm +FLUX ?= flux KUSTOMIZE ?= kustomize CONTROLLER_GEN ?= controller-gen ENVTEST ?= setup-envtest @@ -284,26 +285,20 @@ ENVTEST_K8S_VERSION ?= $(shell \ # Gitea E2E Configuration GITEA_NAMESPACE ?= gitea-e2e GITEA_RELEASE_NAME ?= gitea -GITEA_HELM_REPO_NAME ?= gitea-charts -GITEA_HELM_REPO_URL ?= https://dl.gitea.com/charts/ -GITEA_CHART_NAME ?= gitea -# https://gitea.com/gitea/helm-gitea -GITEA_CHART_VERSION ?= 12.5.0 -GITEA_WAIT_TIMEOUT ?= 300s GITEA_PORT ?= 13000 PROMETHEUS_PORT ?= 19090 # Valkey E2E Configuration VALKEY_NAMESPACE ?= valkey-e2e VALKEY_RELEASE_NAME ?= valkey -VALKEY_HELM_REPO_NAME ?= valkey -VALKEY_HELM_REPO_URL ?= https://valkey.io/valkey-helm -VALKEY_CHART_NAME ?= valkey -# https://valkey.io/valkey-helm/index.yaml -VALKEY_CHART_VERSION ?= 0.9.3 -VALKEY_WAIT_TIMEOUT ?= 120s VALKEY_PORT ?= 16379 +FLUX_NAMESPACE ?= flux-system +FLUX_WAIT_TIMEOUT ?= 300s +FLUX_SERVICES_WAIT_TIMEOUT ?= 600s +FLUX_SERVICES_DIR ?= test/e2e/setup/flux-services +FLUX_SERVICES_SELECTOR ?= e2e.configbutler.io/stack=services + ENVTEST_STAMP = .stamps/envtest-$(ENVTEST_K8S_VERSION).ready .PHONY: setup-envtest @@ -328,36 +323,39 @@ $(CS)/ready: test/e2e/cluster/start-cluster.sh | $(CS) kubectl --context $(CTX) get ns >/dev/null touch $@ -CERT_MANAGER_WAIT_TIMEOUT ?= 300s -CERT_MANAGER_VERSION ?= v1.19.4 -CERT_MANAGER_MANIFEST_URL ?= https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml -$(CS)/cert-manager.installed: $(CS)/ready | $(CS) - kubectl --context $(CTX) apply -f $(CERT_MANAGER_MANIFEST_URL) | grep -v unchanged - kubectl --context $(CTX) -n cert-manager rollout status \ - deploy/cert-manager \ - --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) - kubectl --context $(CTX) -n cert-manager rollout status \ - deploy/cert-manager-webhook \ - --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) - kubectl --context $(CTX) -n cert-manager rollout status \ - deploy/cert-manager-cainjector \ - --timeout=$(CERT_MANAGER_WAIT_TIMEOUT) - echo $(CERT_MANAGER_VERSION) > $@ - -$(CS)/gitea.installed: $(CS)/ready test/e2e/gitea-values.yaml | $(CS) - $(HELM) repo add $(GITEA_HELM_REPO_NAME) $(GITEA_HELM_REPO_URL) 2>/dev/null || true - $(HELM) repo update $(GITEA_HELM_REPO_NAME) - kubectl --context $(CTX) create namespace $(GITEA_NAMESPACE) --dry-run=client -o yaml \ - | kubectl --context $(CTX) apply -f - - $(HELM) --kube-context $(CTX) upgrade --install $(GITEA_RELEASE_NAME) \ - $(GITEA_HELM_REPO_NAME)/$(GITEA_CHART_NAME) \ - --namespace $(GITEA_NAMESPACE) \ - --version $(GITEA_CHART_VERSION) \ - --values test/e2e/gitea-values.yaml - kubectl --context $(CTX) -n $(GITEA_NAMESPACE) rollout status \ - deploy/$(GITEA_RELEASE_NAME) \ - --timeout=$(GITEA_WAIT_TIMEOUT) - echo $(GITEA_CHART_VERSION) > $@ +FLUX_SERVICES_INPUTS = $(shell find $(FLUX_SERVICES_DIR) -type f) +$(CS)/flux.installed: $(CS)/ready | $(CS) + $(FLUX) install --context $(CTX) --namespace $(FLUX_NAMESPACE) + kubectl --context $(CTX) -n $(FLUX_NAMESPACE) wait \ + deployment \ + -l app.kubernetes.io/part-of=flux \ + --for=condition=Available \ + --timeout=$(FLUX_WAIT_TIMEOUT) + kubectl --context $(CTX) wait --for=condition=Established \ + crd/helmrepositories.source.toolkit.fluxcd.io \ + crd/helmreleases.helm.toolkit.fluxcd.io \ + --timeout=$(FLUX_WAIT_TIMEOUT) + $(FLUX) version --client > $@ + +$(CS)/flux-services.ready: $(CS)/flux.installed $$(FLUX_SERVICES_INPUTS) | $(CS) + kubectl --context $(CTX) apply -k $(FLUX_SERVICES_DIR) + count="$$(kubectl --context $(CTX) get helmreleases.helm.toolkit.fluxcd.io \ + --all-namespaces \ + -l $(FLUX_SERVICES_SELECTOR) \ + --no-headers 2>/dev/null | wc -l | tr -d ' ')" + [ "$$count" -gt 0 ] || { echo "ERROR: no Flux-managed e2e HelmRelease resources found" >&2; exit 1; } + if ! kubectl --context $(CTX) wait \ + helmreleases.helm.toolkit.fluxcd.io \ + --all-namespaces \ + -l $(FLUX_SERVICES_SELECTOR) \ + --for=condition=Ready \ + --timeout=$(FLUX_SERVICES_WAIT_TIMEOUT); then + kubectl --context $(CTX) get helmreleases.helm.toolkit.fluxcd.io \ + --all-namespaces \ + -l $(FLUX_SERVICES_SELECTOR) + exit 1 + fi + touch $@ PROMETHEUS_SETUP_MANIFESTS = $(shell find test/e2e/setup/prometheus -type f -name '*.yaml') $(CS)/prometheus.installed: $(CS)/ready hack/e2e/ensure-prometheus-operator.sh $$(PROMETHEUS_SETUP_MANIFESTS) | $(CS) @@ -379,23 +377,8 @@ $(CS)/prometheus.installed: $(CS)/ready hack/e2e/ensure-prometheus-operator.sh $ --timeout=180s touch $@ -$(CS)/valkey.installed: $(CS)/ready test/e2e/valkey-values.yaml | $(CS) - $(HELM) repo add $(VALKEY_HELM_REPO_NAME) $(VALKEY_HELM_REPO_URL) 2>/dev/null || true - $(HELM) repo update $(VALKEY_HELM_REPO_NAME) - kubectl --context $(CTX) create namespace $(VALKEY_NAMESPACE) --dry-run=client -o yaml \ - | kubectl --context $(CTX) apply -f - - $(HELM) --kube-context $(CTX) upgrade --install $(VALKEY_RELEASE_NAME) \ - $(VALKEY_HELM_REPO_NAME)/$(VALKEY_CHART_NAME) \ - --namespace $(VALKEY_NAMESPACE) \ - --version $(VALKEY_CHART_VERSION) \ - --values test/e2e/valkey-values.yaml - kubectl --context $(CTX) -n $(VALKEY_NAMESPACE) rollout status \ - deploy/$(VALKEY_RELEASE_NAME) \ - --timeout=$(VALKEY_WAIT_TIMEOUT) - echo $(VALKEY_CHART_VERSION) > $@ - # Aggregate stamp for external E2E services required by tests. -$(CS)/services.ready: $(CS)/cert-manager.installed $(CS)/prometheus.installed $(CS)/gitea.installed $(CS)/valkey.installed +$(CS)/services.ready: $(CS)/flux-services.ready $(CS)/prometheus.installed touch $@ # Step 1: Generate age key file — no cluster/namespace dependency; safe to run before installation. diff --git a/docs/ci/CI_NON_ROOT_USER_ANALYSIS.md b/docs/ci/CI_NON_ROOT_USER_ANALYSIS.md index b2c8cd522..716019d04 100644 --- a/docs/ci/CI_NON_ROOT_USER_ANALYSIS.md +++ b/docs/ci/CI_NON_ROOT_USER_ANALYSIS.md @@ -186,7 +186,7 @@ If organizational policy requires non-root CI, here's the implementation strateg #### Option 1: Hybrid Approach (Recommended) ```dockerfile # Build stage: root for tool installation -FROM golang:1.26.0 AS ci-builder +FROM golang:1.26.1 AS ci-builder RUN apt-get update && apt-get install... RUN curl -LO kubectl... @@ -200,7 +200,7 @@ USER ciuser #### Option 2: Rootless with Capabilities ```dockerfile # Install tools as root -FROM golang:1.26.0 AS ci +FROM golang:1.26.1 AS ci RUN apt-get update... # Create non-root user with specific capabilities diff --git a/docs/design/flux-e2e-services-plan.md b/docs/design/flux-e2e-services-plan.md new file mode 100644 index 000000000..95e2dc66a --- /dev/null +++ b/docs/design/flux-e2e-services-plan.md @@ -0,0 +1,458 @@ +# Flux-Managed E2E Services Plan + +## Goal + +Reduce e2e setup complexity by making Flux manage the shared support services used by the test cluster. + +Initial scope: + +- `cert-manager` +- `gitea` +- `valkey` + +Out of scope for the first pass: + +- moving the gitops-reverser controller install itself under Flux +- requiring a Git-backed Flux source for e2e bootstrapping +- changing the existing quickstart install-mode coverage (`helm`, `plain-manifests-file`, `config-dir`) + +## Why this direction + +The current e2e bootstrap mixes several orchestration styles: + +- imperative `kubectl apply` for `cert-manager` +- imperative `helm upgrade --install` for `gitea` +- imperative `helm upgrade --install` for `valkey` +- imperative shell setup for Prometheus operator bootstrap +- separate install-mode logic for the system under test + +That works, but it spreads lifecycle logic across the Makefile and shell scripts. + +The cleanest first step is to make Flux own the shared service layer while leaving the system-under-test install +flow unchanged. That gives us: + +- fewer imperative Helm steps in the Makefile +- a single place for service definitions +- explicit dependency ordering via Flux `dependsOn` +- a more GitOps-shaped e2e environment without introducing circular bootstrap problems + +## Recommendation + +Create a single Flux-applied services bundle under: + +`test/e2e/setup/flux-services/` + +That bundle should contain: + +- Flux services manifests +- `HelmRepository` resources for upstream charts +- `HelmRelease` resources for `cert-manager`, `gitea`, and `valkey` +- namespaces for those services +- any values/config needed by those Helm releases + +The Makefile should always apply this bundle as part of cluster services preparation. + +Flux should not initially reconcile from a Git repository. +Instead, the e2e setup should `kubectl apply` the Flux custom resources from the local working tree and then wait for +Flux to reconcile them in-cluster. + +This keeps bootstrap simple while still using Flux as the control plane for service installs. + +## Current seams in the Makefile + +Today the shared services are built around these cluster-scoped stamps: + +- `$(CS)/cert-manager.installed` +- `$(CS)/gitea.installed` +- `$(CS)/valkey.installed` +- `$(CS)/prometheus.installed` +- `$(CS)/services.ready` + +The imperative Helm/install logic currently lives in: + +- `$(CS)/cert-manager.installed` +- `$(CS)/gitea.installed` +- `$(CS)/valkey.installed` + +The system under test depends on `$(CS)/services.ready`, and the Go e2e suite always triggers that path through +`prepare-e2e`. + +That means the Flux migration should preserve the same high-level contract: + +`prepare-e2e` -> shared services ready -> controller install -> image override -> tests + +## Target architecture + +### 1. Shared cluster bootstrap + +Keep a small amount of imperative bootstrap in Make: + +- create/reuse the k3d cluster +- install Flux controllers +- apply the local Flux services bundle +- wait for all Flux-managed `HelmRelease` objects to become Ready + +This replaces direct Helm commands in Make, but it does not try to eliminate all Make orchestration. + +### 2. Flux-owned services bundle + +Create a local manifest tree like: + +```text +test/e2e/setup/flux-services/ + kustomization.yaml + namespaces/ + cert-manager.yaml + gitea-e2e.yaml + valkey-e2e.yaml + repositories/ + jetstack.yaml + gitea.yaml + valkey.yaml + releases/ + cert-manager.yaml + gitea.yaml + valkey.yaml + values/ + gitea-values.yaml + valkey-values.yaml +``` + +Recommended behavior: + +- `cert-manager` installs first +- `gitea` and `valkey` depend only on their repositories and namespaces +- if webhook/certificate timing becomes relevant later, additional `dependsOn` can be added cleanly + +### 3. Minimal Makefile contract + +Everything managed by Flux should disappear as a service-specific Make target. + +The target shape should become: + +- `$(CS)/flux.installed` +- `$(CS)/flux-services.ready` +- `$(CS)/services.ready` + +That means: + +- no `$(CS)/cert-manager.ready` +- no `$(CS)/gitea.ready` +- no `$(CS)/valkey.ready` +- no service-specific install stamps for Flux-owned services + +The Makefile should not know how each service is installed anymore. +It should only know: + +1. Flux is installed. +2. The local Flux bundle was applied. +3. All Flux-owned `HelmRelease` objects are Ready. + +That is the cleanest simplification boundary. + +## Proposed implementation phases + +## Phase 1: Flux for cert-manager, Gitea, and Valkey + +### Desired outcome + +Replace: + +- direct `kubectl apply` of remote cert-manager manifest +- direct Helm install of Gitea +- direct Helm install of Valkey + +With: + +- Flux controllers installed once per e2e cluster context +- local Flux resources applied from `test/e2e/setup/flux-services/` +- one readiness gate based on Flux-managed `HelmRelease` objects + +### Concrete changes + +1. Add a new local Flux services directory. +2. Define service namespaces as plain manifests. +3. Define `HelmRepository` resources for: + - Jetstack + - Gitea charts + - Valkey charts +4. Define `HelmRelease` resources for: + - `cert-manager` + - `gitea` + - `valkey` +5. Reference existing e2e values for Gitea and Valkey, or move those values into the new directory if that makes the + bundle easier to understand. +6. Add `dependsOn` only where it improves determinism; do not over-model dependencies. +7. Replace the old `$(CS)/cert-manager.installed`, `$(CS)/gitea.installed`, and `$(CS)/valkey.installed` recipes with: + - Flux bootstrap + - apply local bundle + - a single `flux-services.ready` wait + +### Readiness strategy + +Use one Make-level wait: + +1. Query all `HelmRelease` objects in the Flux services bundle. +2. Fail unless every one reports `Ready=True`. + +Recommended implementation shape: + +- label all Flux-owned e2e `HelmRelease` objects consistently +- use one `kubectl`-driven readiness check against that label set + +Example direction: + +```bash +kubectl --context "${CTX}" get helmreleases.helm.toolkit.fluxcd.io -A \ + -l e2e.configbutler.io/stack=services +``` + +This keeps the Makefile simple while still leaving detailed failure information in the `HelmRelease` status when +something goes wrong. + +If a specific workload later proves flaky even when the `HelmRelease` is Ready, we can add a focused workload-level +wait at that time. It should not be the default shape of the first implementation. + +### Makefile simplification target + +The Makefile should stop containing: + +- chart repo add/update commands +- per-service `helm upgrade --install` logic +- service-specific inline Helm arguments + +The Makefile should only contain: + +- bootstrap Flux +- apply the local Flux services bundle +- run one readiness check for all Flux-managed `HelmRelease` objects + +That is the main line-count and complexity win. + +## Phase 2: Evaluate Prometheus + +Prometheus is more custom today because it depends on a Prometheus operator setup path. + +Recommendation: + +- keep Prometheus out of the first Flux migration +- once Phase 1 is stable, decide whether to: + - manage the operator with Flux and keep local CRs in place, or + - keep Prometheus imperative if it remains simpler and more reliable + +This avoids mixing one straightforward cleanup with a more open-ended refactor. + +## Phase 3: Optional Flux-managed controller install mode + +Only after the service layer is stable, consider a new install mode for the controller itself. + +Possible future mode: + +- `INSTALL_MODE=flux-helm` + +But this should not be part of the first implementation because the current e2e flow patches the controller Deployment +image after install. A Flux-managed Deployment would likely reconcile that change away unless the image override is +designed into the Flux resources. + +## Why not use a GitRepository source first + +Using Flux without a Git source is the right first step here. + +Reasons: + +- simpler bootstrap +- fewer moving parts +- no circular dependency on Gitea to install Gitea +- easier local debugging +- still exercises Flux controllers and reconciliation loops + +In practice, Flux would reconcile local custom resources we apply directly: + +- `HelmRepository` +- `HelmRelease` +- later possibly `Kustomization` + +If this works well, a later phase can move the service definitions into a Git-backed source in Gitea for higher +fidelity. + +## Proposed file layout + +Recommended initial layout: + +```text +test/e2e/setup/flux-services/ + kustomization.yaml + namespaces/ + cert-manager.yaml + gitea-e2e.yaml + valkey-e2e.yaml + repositories/ + jetstack.yaml + gitea-charts.yaml + valkey.yaml + releases/ + cert-manager.yaml + gitea.yaml + valkey.yaml + values/ + gitea-values.yaml + valkey-values.yaml +``` + +Optional helper location if needed: + +```text +hack/e2e/ + install-flux.sh +``` + +The default preference should be to keep logic in Make small and use at most one focused helper script if readability +improves. + +## Proposed Makefile shape + +### New stamps + +- `$(CS)/flux.installed` +- `$(CS)/flux-services.ready` + +### Updated aggregate + +`$(CS)/services.ready` should depend on: + +- `$(CS)/flux-services.ready` +- existing Prometheus readiness stamp until Prometheus is migrated or intentionally left as-is + +### What should be removed + +Remove or replace: + +- `GITEA_HELM_REPO_*` Make variables +- `VALKEY_HELM_REPO_*` Make variables +- direct Helm install recipes for Gitea and Valkey +- remote cert-manager manifest apply recipe +- per-service readiness stamps for Flux-owned services + +The chart versions should live with the Flux `HelmRelease` resources instead of the Makefile. + +## Flux bootstrap recommendation + +Flux itself should be bootstrapped separately from the services bundle. + +Recommended shape: + +1. Keep one stamp for Flux controller installation: + - `$(CS)/flux.installed` +2. Install Flux controllers into the cluster with the Flux CLI: + - `flux install` +3. Wait for Flux controllers in `flux-system` to be Available. +4. After that, apply `test/e2e/setup/flux-services/`. + +Why separate it from the services bundle: + +- the bundle contains Flux custom resources, so the CRDs/controllers must already exist +- Flux installation changes much less often than the service definitions +- it keeps the failure mode obvious: either Flux itself failed, or a Flux-managed service failed + +Alternative: + +- commit raw Flux install manifests into the repo and apply them with `kubectl` + +That works too, but the Flux CLI is the cleaner bootstrap tool for local e2e because it gives a short, supported way to +install the controllers without carrying a large generated manifest in-tree. + +## Flux CLI in the devcontainer + +Yes, the Flux CLI should be installed in the devcontainer if we adopt this plan. + +Reasoning: + +- the Makefile will depend on it for `$(CS)/flux.installed` +- local e2e and CI/devcontainer behavior should match +- Flux bootstrap is easier and less noisy through `flux install` than by vendoring install YAML + +Suggested approach: + +- add `FLUX_VERSION` alongside the other tool versions in `.devcontainer/Dockerfile` +- install the `flux` binary in the same style as `kubectl`, `helm`, and `k3d` + +What I would not do: + +- require developers to install Flux manually outside the container +- make the e2e bootstrap fetch and pipe an ad-hoc shell installer every run + +If we later decide to avoid the Flux CLI entirely, we can switch to checked-in install manifests and remove the binary. +But for the initial implementation, shipping the CLI in the devcontainer is the simpler and cleaner choice. + +## Cleanup model + +Cleanup should stay simple. + +Recommended approach: + +- keep Flux controllers installed for the cluster context once created +- treat Flux-managed services as shared cluster-scoped e2e prerequisites +- continue cleaning only per-test install namespaces during normal install cleanup +- add a dedicated target if full service teardown is needed + +This matches the current stamp philosophy of reusing expensive cluster setup work. + +## Risks and mitigations + +### Risk: readiness becomes less obvious + +Mitigation: + +- use one labeled `HelmRelease` readiness check +- inspect `HelmRelease` status and events when debugging failures + +### Risk: too much abstraction hides failures + +Mitigation: + +- keep manifests local and readable +- do not generate this bundle dynamically +- prefer explicit YAML over large shell wrappers + +### Risk: cert-manager ordering issues + +Mitigation: + +- install cert-manager through its own `HelmRelease` +- use `dependsOn` only where useful +- keep workload waits explicit in Make + +### Risk: trying to migrate too much at once + +Mitigation: + +- do cert-manager, Gitea, and Valkey first +- postpone Prometheus and controller install-mode changes + +## Success criteria + +Phase 1 is successful when: + +1. The Makefile has materially fewer service-install lines and less inline Helm logic. +2. `prepare-e2e` still remains the single entrypoint used by the Go suite. +3. Shared services are reconciled by Flux from local manifests. +4. `make test-e2e`, `make test-e2e-quickstart-manifest`, and `make test-e2e-quickstart-helm` still pass. +5. Local debugging remains straightforward: inspect Flux `HelmRelease` objects first, then workloads if needed. + +## Suggested execution order + +1. Add Flux bootstrap target and stamps. +2. Add `test/e2e/setup/flux-services/` with `HelmRepository` and `HelmRelease` YAMLs. +3. Migrate `cert-manager` into Flux. +4. Migrate `gitea` into Flux. +5. Migrate `valkey` into Flux. +6. Rewire `$(CS)/services.ready` to depend on one `flux-services.ready` target. +7. Remove old imperative Helm/install logic from the Makefile. +8. Run full validation. + +## Notes for implementation + +- Keep the implementation aggressive about deleting now-redundant Makefile code. +- Prefer one shared services folder over per-service bootstrap patterns. +- Prefer stable, checked-in YAML over dynamically rendered e2e service manifests. +- Keep the controller install path separate until image override semantics are redesigned for Flux ownership. diff --git a/go.mod b/go.mod index 1107015c7..01234b010 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ConfigButler/gitops-reverser -go 1.26.0 +go 1.26.1 require ( filippo.io/age v1.3.1 diff --git a/test/e2e/setup/flux-services/kustomization.yaml b/test/e2e/setup/flux-services/kustomization.yaml new file mode 100644 index 000000000..6e166fed3 --- /dev/null +++ b/test/e2e/setup/flux-services/kustomization.yaml @@ -0,0 +1,33 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +labels: + - includeSelectors: false + pairs: + e2e.configbutler.io/stack: services + +resources: + - namespaces/cert-manager.yaml + - namespaces/gitea-e2e.yaml + - namespaces/valkey-e2e.yaml + - repositories/jetstack.yaml + - repositories/gitea-charts.yaml + - repositories/valkey.yaml + - releases/cert-manager.yaml + - releases/gitea.yaml + - releases/valkey.yaml + +generatorOptions: + disableNameSuffixHash: true + labels: + reconcile.fluxcd.io/watch: Enabled + +configMapGenerator: + - name: gitea-values + namespace: gitea-e2e + files: + - values.yaml=values/gitea-values.yaml + - name: valkey-values + namespace: valkey-e2e + files: + - values.yaml=values/valkey-values.yaml diff --git a/test/e2e/setup/flux-services/namespaces/cert-manager.yaml b/test/e2e/setup/flux-services/namespaces/cert-manager.yaml new file mode 100644 index 000000000..c90416ff4 --- /dev/null +++ b/test/e2e/setup/flux-services/namespaces/cert-manager.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cert-manager diff --git a/test/e2e/setup/flux-services/namespaces/gitea-e2e.yaml b/test/e2e/setup/flux-services/namespaces/gitea-e2e.yaml new file mode 100644 index 000000000..4db3be45c --- /dev/null +++ b/test/e2e/setup/flux-services/namespaces/gitea-e2e.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gitea-e2e diff --git a/test/e2e/setup/flux-services/namespaces/valkey-e2e.yaml b/test/e2e/setup/flux-services/namespaces/valkey-e2e.yaml new file mode 100644 index 000000000..3ee277f35 --- /dev/null +++ b/test/e2e/setup/flux-services/namespaces/valkey-e2e.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: valkey-e2e diff --git a/test/e2e/setup/flux-services/releases/cert-manager.yaml b/test/e2e/setup/flux-services/releases/cert-manager.yaml new file mode 100644 index 000000000..5436a4307 --- /dev/null +++ b/test/e2e/setup/flux-services/releases/cert-manager.yaml @@ -0,0 +1,28 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: cert-manager + namespace: cert-manager +spec: + interval: 30m + timeout: 5m + chart: + spec: + chart: cert-manager + sourceRef: + kind: HelmRepository + name: jetstack + namespace: flux-system + version: v1.19.4 + install: + createNamespace: false + crds: Create + remediation: + retries: 3 + upgrade: + crds: CreateReplace + remediation: + retries: 3 + values: + crds: + enabled: true diff --git a/test/e2e/setup/flux-services/releases/gitea.yaml b/test/e2e/setup/flux-services/releases/gitea.yaml new file mode 100644 index 000000000..51b6f9c3d --- /dev/null +++ b/test/e2e/setup/flux-services/releases/gitea.yaml @@ -0,0 +1,27 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: gitea + namespace: gitea-e2e +spec: + interval: 30m + timeout: 5m + chart: + spec: + chart: gitea + sourceRef: + kind: HelmRepository + name: gitea-charts + namespace: flux-system + version: 12.5.0 + install: + createNamespace: false + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: gitea-values + valuesKey: values.yaml diff --git a/test/e2e/setup/flux-services/releases/valkey.yaml b/test/e2e/setup/flux-services/releases/valkey.yaml new file mode 100644 index 000000000..3efe04eee --- /dev/null +++ b/test/e2e/setup/flux-services/releases/valkey.yaml @@ -0,0 +1,27 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: valkey + namespace: valkey-e2e +spec: + interval: 30m + timeout: 5m + chart: + spec: + chart: valkey + sourceRef: + kind: HelmRepository + name: valkey + namespace: flux-system + version: 0.9.3 + install: + createNamespace: false + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: valkey-values + valuesKey: values.yaml diff --git a/test/e2e/setup/flux-services/repositories/gitea-charts.yaml b/test/e2e/setup/flux-services/repositories/gitea-charts.yaml new file mode 100644 index 000000000..9ad271373 --- /dev/null +++ b/test/e2e/setup/flux-services/repositories/gitea-charts.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: gitea-charts + namespace: flux-system +spec: + interval: 30m + url: https://dl.gitea.com/charts/ diff --git a/test/e2e/setup/flux-services/repositories/jetstack.yaml b/test/e2e/setup/flux-services/repositories/jetstack.yaml new file mode 100644 index 000000000..0dd08be97 --- /dev/null +++ b/test/e2e/setup/flux-services/repositories/jetstack.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: jetstack + namespace: flux-system +spec: + interval: 30m + url: https://charts.jetstack.io diff --git a/test/e2e/setup/flux-services/repositories/valkey.yaml b/test/e2e/setup/flux-services/repositories/valkey.yaml new file mode 100644 index 000000000..18838fadb --- /dev/null +++ b/test/e2e/setup/flux-services/repositories/valkey.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: valkey + namespace: flux-system +spec: + interval: 30m + url: https://valkey.io/valkey-helm diff --git a/test/e2e/gitea-values.yaml b/test/e2e/setup/flux-services/values/gitea-values.yaml similarity index 70% rename from test/e2e/gitea-values.yaml rename to test/e2e/setup/flux-services/values/gitea-values.yaml index 551c7c747..7afb2b7e2 100644 --- a/test/e2e/gitea-values.yaml +++ b/test/e2e/setup/flux-services/values/gitea-values.yaml @@ -1,11 +1,9 @@ -# Gitea Helm values for e2e testing -# Optimized for speed and CI environments +# Gitea Helm values for e2e testing. +# Optimized for speed and CI environments. -# Disable persistence for fast, ephemeral testing persistence: enabled: false -# Disable external databases - use SQLite in memory memcached: enabled: false postgresql-ha: @@ -19,12 +17,9 @@ mysql: gitea: config: - # Database configuration - SQLite file for persistence between init containers database: DB_TYPE: sqlite3 PATH: "/data/gitea/gitea.db" - - # Server configuration server: PROTOCOL: http HTTP_PORT: 13000 @@ -33,22 +28,14 @@ gitea: SSH_DOMAIN: gitea-ssh.gitea-e2e.svc.cluster.local SSH_PORT: 22 START_SSH_SERVER: true - - # Security settings security: INSTALL_LOCK: true SECRET_KEY: "e2e-test-secret-key-not-for-production" - - # Service settings service: DISABLE_REGISTRATION: true DEFAULT_KEEP_EMAIL_PRIVATE: true - - # Repository settings repository: DEFAULT_BRANCH: main - - # Additional settings for e2e indexer: ISSUE_INDEXER_TYPE: bleve REPO_INDEXER_ENABLED: false @@ -56,14 +43,11 @@ gitea: ENABLED: false webhook: ALLOWED_HOST_LIST: "*" - - # Pre-configured admin user for API access admin: username: giteaadmin password: giteapassword123 email: admin@example.com -# Service configuration service: http: type: ClusterIP @@ -72,12 +56,11 @@ service: type: ClusterIP port: 22 -# Resource limits for CI environments resources: limits: memory: "1024Mi" requests: - memory: "512Mi" + memory: "512Mi" podSecurityContext: runAsNonRoot: true diff --git a/test/e2e/valkey-values.yaml b/test/e2e/setup/flux-services/values/valkey-values.yaml similarity index 100% rename from test/e2e/valkey-values.yaml rename to test/e2e/setup/flux-services/values/valkey-values.yaml From 816e814e7705480bd42880a49166d031e0813213 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 13 Mar 2026 09:52:54 +0000 Subject: [PATCH 5/8] chore: Also doing prometheus, intersting take to install it with the kube-prometheus-stack --- Makefile | 31 +----- docs/design/flux-e2e-services-plan.md | 79 +++++++++++++-- hack/e2e/setup-port-forwards.sh | 5 +- .../setup/flux-services/kustomization.yaml | 7 ++ .../namespaces/prometheus-operator.yaml | 4 + .../flux-services/releases/prometheus.yaml | 29 ++++++ .../repositories/prometheus-community.yaml | 8 ++ .../values/prometheus-values.yaml | 98 +++++++++++++++++++ 8 files changed, 222 insertions(+), 39 deletions(-) create mode 100644 test/e2e/setup/flux-services/namespaces/prometheus-operator.yaml create mode 100644 test/e2e/setup/flux-services/releases/prometheus.yaml create mode 100644 test/e2e/setup/flux-services/repositories/prometheus-community.yaml create mode 100644 test/e2e/setup/flux-services/values/prometheus-values.yaml diff --git a/Makefile b/Makefile index a7129170d..3480c1e7c 100644 --- a/Makefile +++ b/Makefile @@ -283,14 +283,10 @@ ENVTEST_K8S_VERSION ?= $(shell \ ) # Gitea E2E Configuration -GITEA_NAMESPACE ?= gitea-e2e -GITEA_RELEASE_NAME ?= gitea GITEA_PORT ?= 13000 PROMETHEUS_PORT ?= 19090 # Valkey E2E Configuration -VALKEY_NAMESPACE ?= valkey-e2e -VALKEY_RELEASE_NAME ?= valkey VALKEY_PORT ?= 16379 FLUX_NAMESPACE ?= flux-system @@ -337,7 +333,8 @@ $(CS)/flux.installed: $(CS)/ready | $(CS) --timeout=$(FLUX_WAIT_TIMEOUT) $(FLUX) version --client > $@ -$(CS)/flux-services.ready: $(CS)/flux.installed $$(FLUX_SERVICES_INPUTS) | $(CS) +# Aggregate stamp for external E2E services required by tests. +$(CS)/services.ready: $(CS)/flux.installed $$(FLUX_SERVICES_INPUTS) | $(CS) kubectl --context $(CTX) apply -k $(FLUX_SERVICES_DIR) count="$$(kubectl --context $(CTX) get helmreleases.helm.toolkit.fluxcd.io \ --all-namespaces \ @@ -357,30 +354,6 @@ $(CS)/flux-services.ready: $(CS)/flux.installed $$(FLUX_SERVICES_INPUTS) | $(CS) fi touch $@ -PROMETHEUS_SETUP_MANIFESTS = $(shell find test/e2e/setup/prometheus -type f -name '*.yaml') -$(CS)/prometheus.installed: $(CS)/ready hack/e2e/ensure-prometheus-operator.sh $$(PROMETHEUS_SETUP_MANIFESTS) | $(CS) - export KUBECONTEXT=$(CTX) - bash hack/e2e/ensure-prometheus-operator.sh - kubectl --context $(CTX) wait --for=condition=Established \ - crd/prometheuses.monitoring.coreos.com \ - crd/servicemonitors.monitoring.coreos.com \ - --timeout=180s - kubectl --context $(CTX) apply -n prometheus-operator -f test/e2e/setup/prometheus - kubectl --context $(CTX) wait --for=condition=Available \ - prometheus/prometheus-shared-e2e \ - -n prometheus-operator \ - --timeout=180s - kubectl --context $(CTX) wait --for=condition=ready \ - pod \ - -l prometheus=prometheus-shared-e2e \ - -n prometheus-operator \ - --timeout=180s - touch $@ - -# Aggregate stamp for external E2E services required by tests. -$(CS)/services.ready: $(CS)/flux-services.ready $(CS)/prometheus.installed - touch $@ - # Step 1: Generate age key file — no cluster/namespace dependency; safe to run before installation. $(CS)/age-key.txt: Makefile test/e2e/tools/gen-age-key/main.go | $(CS) go run ./test/e2e/tools/gen-age-key \ diff --git a/docs/design/flux-e2e-services-plan.md b/docs/design/flux-e2e-services-plan.md index 95e2dc66a..2bb9c354b 100644 --- a/docs/design/flux-e2e-services-plan.md +++ b/docs/design/flux-e2e-services-plan.md @@ -1,5 +1,21 @@ # Flux-Managed E2E Services Plan +## Status update + +The first Flux migration is now in place. + +Completed: + +- Flux controllers are bootstrapped from the Makefile +- shared e2e services are applied from `test/e2e/setup/flux-services/` +- `cert-manager`, `gitea`, `prometheus`, and `valkey` are reconciled through Flux `HelmRelease` resources +- `$(CS)/services.ready` is now the single shared-services readiness gate + +Still intentionally outside that migration: + +- the controller install itself +- Git-backed Flux sources + ## Goal Reduce e2e setup complexity by making Flux manage the shared support services used by the test cluster. @@ -131,7 +147,6 @@ Everything managed by Flux should disappear as a service-specific Make target. The target shape should become: - `$(CS)/flux.installed` -- `$(CS)/flux-services.ready` - `$(CS)/services.ready` That means: @@ -186,7 +201,7 @@ With: 7. Replace the old `$(CS)/cert-manager.installed`, `$(CS)/gitea.installed`, and `$(CS)/valkey.installed` recipes with: - Flux bootstrap - apply local bundle - - a single `flux-services.ready` wait + - a single `services.ready` wait ### Readiness strategy @@ -242,6 +257,57 @@ Recommendation: This avoids mixing one straightforward cleanup with a more open-ended refactor. +## Completed step: replace `$(CS)/prometheus.installed` + +This increment replaced the old Prometheus-specific stamp target in the Makefile. + +### Scope + +In scope: + +- remove the special-case `$(CS)/prometheus.installed` install path from the Makefile +- move Prometheus bootstrap behind the same Flux-managed service pattern already used for `cert-manager`, `gitea`, and + `valkey` +- preserve the existing shared-service contract of `$(CS)/services.ready` + +Out of scope: + +- changing Prometheus scrape behavior +- redesigning test port-forwarding +- migrating the controller install under Flux +- introducing a Git-backed Flux source + +### Recommended shape + +Add Prometheus as another local Flux-managed service under `test/e2e/setup/flux-services/`. + +That likely means: + +- a `HelmRepository` for the Prometheus operator chart source +- a `HelmRelease` for the operator stack +- local manifests or Flux-managed resources for the e2e-specific Prometheus instance and `ServiceMonitor` +- the same shared `HelmRelease` readiness gate used by `$(CS)/services.ready` + +The key design goal is that `$(CS)/services.ready` should depend on one shared Flux services gate, not on a separate +Prometheus install recipe. + +### Makefile target end state + +After this step, the preferred shape is: + +- keep `$(CS)/flux.installed` +- remove `$(CS)/prometheus.installed` +- keep `$(CS)/services.ready` as the single shared-services readiness target + +### Acceptance criteria for this step + +This step is done when: + +1. Prometheus no longer has a dedicated install recipe in `Makefile`. +2. `prepare-e2e` still reaches a working shared Prometheus endpoint for the existing e2e flow. +3. The full shared-service bootstrap path is expressed through Flux-applied local manifests. +4. `make test-e2e`, `make test-e2e-quickstart-manifest`, and `make test-e2e-quickstart-helm` still pass. + ## Phase 3: Optional Flux-managed controller install mode Only after the service layer is stable, consider a new install mode for the controller itself. @@ -314,14 +380,11 @@ improves. ### New stamps - `$(CS)/flux.installed` -- `$(CS)/flux-services.ready` +- `$(CS)/services.ready` ### Updated aggregate -`$(CS)/services.ready` should depend on: - -- `$(CS)/flux-services.ready` -- existing Prometheus readiness stamp until Prometheus is migrated or intentionally left as-is +`$(CS)/services.ready` should remain the single readiness stamp for shared e2e services. ### What should be removed @@ -446,7 +509,7 @@ Phase 1 is successful when: 3. Migrate `cert-manager` into Flux. 4. Migrate `gitea` into Flux. 5. Migrate `valkey` into Flux. -6. Rewire `$(CS)/services.ready` to depend on one `flux-services.ready` target. +6. Rewire `$(CS)/services.ready` to be the single shared-services readiness target. 7. Remove old imperative Helm/install logic from the Makefile. 8. Run full validation. diff --git a/hack/e2e/setup-port-forwards.sh b/hack/e2e/setup-port-forwards.sh index ffcbc33cd..cb6ced0de 100755 --- a/hack/e2e/setup-port-forwards.sh +++ b/hack/e2e/setup-port-forwards.sh @@ -5,6 +5,7 @@ set -euo pipefail GITEA_NAMESPACE=${GITEA_NAMESPACE:-gitea-e2e} PROMETHEUS_NAMESPACE=${PROMETHEUS_NAMESPACE:-prometheus-operator} PROMETHEUS_INSTANCE_NAME=${PROMETHEUS_INSTANCE_NAME:-prometheus-shared-e2e} +PROMETHEUS_SERVICE=${PROMETHEUS_SERVICE:-prometheus-shared-e2e-prometheus} VALKEY_NAMESPACE=${VALKEY_NAMESPACE:-valkey-e2e} VALKEY_RELEASE_NAME=${VALKEY_RELEASE_NAME:-valkey} KUBE_CONTEXT=${E2E_KUBECONTEXT:-${CTX:-${KUBECONTEXT:-}}} @@ -27,7 +28,7 @@ has_expected_forward_processes() { local valkey_pf gitea_pf="$(ps -ef | grep -E "kubectl( |.* )--context ${KUBE_CONTEXT}( |.* )port-forward( |.* )svc/${GITEA_SERVICE:-gitea-http}( |.* )${GITEA_PORT}:${GITEA_PORT}" | grep -v grep || true)" - prom_pf="$(ps -ef | grep -E "kubectl( |.* )--context ${KUBE_CONTEXT}( |.* )port-forward( |.* )svc/prometheus-operated( |.* )${PROMETHEUS_PORT}:9090" | grep -v grep || true)" + prom_pf="$(ps -ef | grep -E "kubectl( |.* )--context ${KUBE_CONTEXT}( |.* )port-forward( |.* )svc/${PROMETHEUS_SERVICE}( |.* )${PROMETHEUS_PORT}:9090" | grep -v grep || true)" valkey_pf="$(ps -ef | grep -E "kubectl( |.* )--context ${KUBE_CONTEXT}( |.* )port-forward( |.* )svc/${VALKEY_RELEASE_NAME}( |.* )${VALKEY_PORT}:6379" | grep -v grep || true)" [[ -n "${gitea_pf}" && -n "${prom_pf}" && -n "${valkey_pf}" ]] @@ -141,7 +142,7 @@ setup_port_forward() { } # Setup port-forwards -setup_port_forward "Prometheus" "$PROMETHEUS_NAMESPACE" "prometheus-operated" "${PROMETHEUS_PORT}" "9090" +setup_port_forward "Prometheus" "$PROMETHEUS_NAMESPACE" "${PROMETHEUS_SERVICE}" "${PROMETHEUS_PORT}" "9090" setup_port_forward "Gitea" "$GITEA_NAMESPACE" "gitea-http" "${GITEA_PORT}" "${GITEA_PORT}" setup_port_forward "Valkey" "$VALKEY_NAMESPACE" "${VALKEY_RELEASE_NAME}" "${VALKEY_PORT}" "6379" diff --git a/test/e2e/setup/flux-services/kustomization.yaml b/test/e2e/setup/flux-services/kustomization.yaml index 6e166fed3..4355ad930 100644 --- a/test/e2e/setup/flux-services/kustomization.yaml +++ b/test/e2e/setup/flux-services/kustomization.yaml @@ -9,12 +9,15 @@ labels: resources: - namespaces/cert-manager.yaml - namespaces/gitea-e2e.yaml + - namespaces/prometheus-operator.yaml - namespaces/valkey-e2e.yaml - repositories/jetstack.yaml - repositories/gitea-charts.yaml + - repositories/prometheus-community.yaml - repositories/valkey.yaml - releases/cert-manager.yaml - releases/gitea.yaml + - releases/prometheus.yaml - releases/valkey.yaml generatorOptions: @@ -27,6 +30,10 @@ configMapGenerator: namespace: gitea-e2e files: - values.yaml=values/gitea-values.yaml + - name: prometheus-values + namespace: prometheus-operator + files: + - values.yaml=values/prometheus-values.yaml - name: valkey-values namespace: valkey-e2e files: diff --git a/test/e2e/setup/flux-services/namespaces/prometheus-operator.yaml b/test/e2e/setup/flux-services/namespaces/prometheus-operator.yaml new file mode 100644 index 000000000..a5867495f --- /dev/null +++ b/test/e2e/setup/flux-services/namespaces/prometheus-operator.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: prometheus-operator diff --git a/test/e2e/setup/flux-services/releases/prometheus.yaml b/test/e2e/setup/flux-services/releases/prometheus.yaml new file mode 100644 index 000000000..8464611fa --- /dev/null +++ b/test/e2e/setup/flux-services/releases/prometheus.yaml @@ -0,0 +1,29 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: prometheus-shared-e2e + namespace: prometheus-operator +spec: + interval: 30m + timeout: 10m + chart: + spec: + chart: kube-prometheus-stack + sourceRef: + kind: HelmRepository + name: prometheus-community + namespace: flux-system + version: 82.10.3 + install: + createNamespace: false + crds: Create + remediation: + retries: 3 + upgrade: + crds: CreateReplace + remediation: + retries: 3 + valuesFrom: + - kind: ConfigMap + name: prometheus-values + valuesKey: values.yaml diff --git a/test/e2e/setup/flux-services/repositories/prometheus-community.yaml b/test/e2e/setup/flux-services/repositories/prometheus-community.yaml new file mode 100644 index 000000000..f5696aa92 --- /dev/null +++ b/test/e2e/setup/flux-services/repositories/prometheus-community.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: prometheus-community + namespace: flux-system +spec: + interval: 30m + url: https://prometheus-community.github.io/helm-charts diff --git a/test/e2e/setup/flux-services/values/prometheus-values.yaml b/test/e2e/setup/flux-services/values/prometheus-values.yaml new file mode 100644 index 000000000..b8f3d25cd --- /dev/null +++ b/test/e2e/setup/flux-services/values/prometheus-values.yaml @@ -0,0 +1,98 @@ +# kube-prometheus-stack values for the shared e2e Prometheus instance. +# Keep this operator install small and deterministic for local/CI bootstrap. + +fullnameOverride: prometheus-shared-e2e +nameOverride: prometheus-shared-e2e +cleanPrometheusOperatorObjectNames: true + +defaultRules: + create: false + +alertmanager: + enabled: false + +grafana: + enabled: false + +kubeApiServer: + enabled: false + +kubeControllerManager: + enabled: false + +coreDns: + enabled: false + +kubeEtcd: + enabled: false + +kubelet: + enabled: false + +kubeProxy: + enabled: false + +kubeScheduler: + enabled: false + +kubeStateMetrics: + enabled: false + +nodeExporter: + enabled: false + +prometheus-node-exporter: + enabled: false + +prometheus-windows-exporter: + enabled: false + +prometheus: + serviceAccount: + create: true + name: prometheus-shared-e2e + prometheusSpec: + replicas: 1 + retention: 2h + scrapeInterval: 5s + evaluationInterval: 5s + serviceMonitorSelectorNilUsesHelmValues: false + serviceMonitorSelector: + matchLabels: + monitoring.configbutler.ai/instance: gitops-reverser + serviceMonitorNamespaceSelector: {} + podMonitorSelectorNilUsesHelmValues: false + podMonitorSelector: {} + podMonitorNamespaceSelector: {} + ruleSelectorNilUsesHelmValues: false + ruleSelector: {} + ruleNamespaceSelector: {} + probeSelectorNilUsesHelmValues: false + probeSelector: {} + probeNamespaceSelector: {} + scrapeConfigSelectorNilUsesHelmValues: false + scrapeConfigSelector: {} + scrapeConfigNamespaceSelector: {} + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + additionalServiceMonitors: + - name: gitops-reverser + additionalLabels: + monitoring.configbutler.ai/instance: gitops-reverser + jobLabel: app.kubernetes.io/name + namespaceSelector: + any: true + selector: + matchLabels: + app.kubernetes.io/name: gitops-reverser + endpoints: + - port: metrics + path: /metrics + scheme: http + interval: 5s + scrapeTimeout: 4s From 1a00106ff1639dad5086210c03244dbd463cc01b Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 13 Mar 2026 10:52:53 +0000 Subject: [PATCH 6/8] chore: Refactor install steps to not rely the huge kube-prometheus-stack --- Makefile | 33 +++---- docs/ai/MAKEFILE_TARGETS.md | 2 +- docs/design/flux-e2e-services-plan.md | 48 ++++----- hack/e2e/ensure-prometheus-operator.sh | 55 ----------- hack/e2e/setup-port-forwards.sh | 2 +- .../setup/flux-services/kustomization.yaml | 8 +- .../releases/prometheus-operator.yaml | 15 +++ .../flux-services/releases/prometheus.yaml | 29 ------ .../repositories/prometheus-community.yaml | 8 -- .../repositories/prometheus-operator.yaml | 10 ++ .../values/prometheus-values.yaml | 98 ------------------- test/e2e/setup/manifests/README.md | 7 ++ .../prometheus/instance.yaml} | 2 + .../prometheus/prometheus-rbac.yaml | 1 + .../prometheus/servicemonitor.yaml | 0 15 files changed, 79 insertions(+), 239 deletions(-) delete mode 100755 hack/e2e/ensure-prometheus-operator.sh create mode 100644 test/e2e/setup/flux-services/releases/prometheus-operator.yaml delete mode 100644 test/e2e/setup/flux-services/releases/prometheus.yaml delete mode 100644 test/e2e/setup/flux-services/repositories/prometheus-community.yaml create mode 100644 test/e2e/setup/flux-services/repositories/prometheus-operator.yaml delete mode 100644 test/e2e/setup/flux-services/values/prometheus-values.yaml create mode 100644 test/e2e/setup/manifests/README.md rename test/e2e/setup/{prometheus/prometheus.yaml => manifests/prometheus/instance.yaml} (90%) rename test/e2e/setup/{ => manifests}/prometheus/prometheus-rbac.yaml (96%) rename test/e2e/setup/{ => manifests}/prometheus/servicemonitor.yaml (100%) diff --git a/Makefile b/Makefile index 3480c1e7c..899280d93 100644 --- a/Makefile +++ b/Makefile @@ -320,6 +320,10 @@ $(CS)/ready: test/e2e/cluster/start-cluster.sh | $(CS) touch $@ FLUX_SERVICES_INPUTS = $(shell find $(FLUX_SERVICES_DIR) -type f) +LAST_STEP_MANIFESTS_DIR = test/e2e/setup/manifests +LAST_STEP_MANIFESTS = $(shell find $(LAST_STEP_MANIFESTS_DIR) -type f -name '*.yaml') +FLUX_SERVICES_READY_RESOURCE_TYPES = helmreleases.helm.toolkit.fluxcd.io,kustomizations.kustomize.toolkit.fluxcd.io +SERVICES_READY_INPUTS = $(CS)/flux.installed $$(FLUX_SERVICES_INPUTS) $$(LAST_STEP_MANIFESTS) $(CS)/flux.installed: $(CS)/ready | $(CS) $(FLUX) install --context $(CTX) --namespace $(FLUX_NAMESPACE) kubectl --context $(CTX) -n $(FLUX_NAMESPACE) wait \ @@ -334,24 +338,19 @@ $(CS)/flux.installed: $(CS)/ready | $(CS) $(FLUX) version --client > $@ # Aggregate stamp for external E2E services required by tests. -$(CS)/services.ready: $(CS)/flux.installed $$(FLUX_SERVICES_INPUTS) | $(CS) +$(CS)/services.ready: $(SERVICES_READY_INPUTS) | $(CS) + kubectl --context $(CTX) delete helmreleases.helm.toolkit.fluxcd.io/prometheus-shared-e2e -n prometheus-operator --ignore-not-found=true + kubectl --context $(CTX) delete helmrepositories.source.toolkit.fluxcd.io/prometheus-community -n flux-system --ignore-not-found=true + kubectl --context $(CTX) -n prometheus-operator delete prometheus.monitoring.coreos.com/prometheus-shared-e2e servicemonitor.monitoring.coreos.com/gitops-reverser --ignore-not-found=true + kubectl --context $(CTX) -n prometheus-operator delete deployment/prometheus-shared-e2e-operator statefulset/prometheus-prometheus-shared-e2e service/prometheus-shared-e2e-operator service/prometheus-shared-e2e-prometheus --ignore-not-found=true + kubectl --context $(CTX) -n prometheus-operator delete pod/prometheus-prometheus-shared-e2e-0 --ignore-not-found=true --force --grace-period=0 kubectl --context $(CTX) apply -k $(FLUX_SERVICES_DIR) - count="$$(kubectl --context $(CTX) get helmreleases.helm.toolkit.fluxcd.io \ - --all-namespaces \ - -l $(FLUX_SERVICES_SELECTOR) \ - --no-headers 2>/dev/null | wc -l | tr -d ' ')" - [ "$$count" -gt 0 ] || { echo "ERROR: no Flux-managed e2e HelmRelease resources found" >&2; exit 1; } - if ! kubectl --context $(CTX) wait \ - helmreleases.helm.toolkit.fluxcd.io \ - --all-namespaces \ - -l $(FLUX_SERVICES_SELECTOR) \ - --for=condition=Ready \ - --timeout=$(FLUX_SERVICES_WAIT_TIMEOUT); then - kubectl --context $(CTX) get helmreleases.helm.toolkit.fluxcd.io \ - --all-namespaces \ - -l $(FLUX_SERVICES_SELECTOR) - exit 1 - fi + flux_ready_count="$$(kubectl --context $(CTX) get $(FLUX_SERVICES_READY_RESOURCE_TYPES) --all-namespaces -l $(FLUX_SERVICES_SELECTOR) --no-headers 2>/dev/null | wc -l | tr -d ' ')" + [ "$$flux_ready_count" -gt 0 ] || { echo "ERROR: no Flux-managed e2e ready-check resources found" >&2; exit 1; } + if ! kubectl --context $(CTX) wait $(FLUX_SERVICES_READY_RESOURCE_TYPES) --all-namespaces -l $(FLUX_SERVICES_SELECTOR) --for=condition=Ready --timeout=$(FLUX_SERVICES_WAIT_TIMEOUT); then kubectl --context $(CTX) get $(FLUX_SERVICES_READY_RESOURCE_TYPES) --all-namespaces -l $(FLUX_SERVICES_SELECTOR); exit 1; fi + kubectl --context $(CTX) wait --for=condition=Established crd/prometheuses.monitoring.coreos.com crd/servicemonitors.monitoring.coreos.com --timeout=180s + + kubectl --context $(CTX) apply -R -f $(LAST_STEP_MANIFESTS_DIR) touch $@ # Step 1: Generate age key file — no cluster/namespace dependency; safe to run before installation. diff --git a/docs/ai/MAKEFILE_TARGETS.md b/docs/ai/MAKEFILE_TARGETS.md index 9c40af8b0..e3fdf9839 100644 --- a/docs/ai/MAKEFILE_TARGETS.md +++ b/docs/ai/MAKEFILE_TARGETS.md @@ -178,7 +178,7 @@ flowchart TD start_sh(["test/e2e/cluster/start-cluster.sh"]):::src gitea_vals(["test/e2e/gitea-values.yaml"]):::src prom_sh(["hack/e2e/ensure-prometheus-operator.sh"]):::src - prom_manifests(["test/e2e/setup/prometheus/*.yaml"]):::src + prom_manifests(["test/e2e/setup/manifests/**/*.yaml"]):::src cluster_ready[".stamps/cluster/CTX/ready"]:::stamp cert_installed[".stamps/cluster/CTX/\ncert-manager.installed"]:::stamp diff --git a/docs/design/flux-e2e-services-plan.md b/docs/design/flux-e2e-services-plan.md index 2bb9c354b..61f9df5ae 100644 --- a/docs/design/flux-e2e-services-plan.md +++ b/docs/design/flux-e2e-services-plan.md @@ -8,7 +8,9 @@ Completed: - Flux controllers are bootstrapped from the Makefile - shared e2e services are applied from `test/e2e/setup/flux-services/` -- `cert-manager`, `gitea`, `prometheus`, and `valkey` are reconciled through Flux `HelmRelease` resources +- `cert-manager`, `gitea`, and `valkey` are reconciled through Flux `HelmRelease` resources +- the Prometheus Operator is reconciled through a Flux `GitRepository` + `Kustomization` +- a final local manifests step applies the remaining e2e-specific resources after the operator CRDs are ready - `$(CS)/services.ready` is now the single shared-services readiness gate Still intentionally outside that migration: @@ -244,18 +246,18 @@ The Makefile should only contain: That is the main line-count and complexity win. -## Phase 2: Evaluate Prometheus +## Phase 2: Prometheus via Flux-managed operator -Prometheus is more custom today because it depends on a Prometheus operator setup path. +Prometheus remains slightly different from the other shared services because it depends on CRDs and an operator-managed +control loop. Recommendation: -- keep Prometheus out of the first Flux migration -- once Phase 1 is stable, decide whether to: - - manage the operator with Flux and keep local CRs in place, or - - keep Prometheus imperative if it remains simpler and more reliable +- install the Prometheus Operator from upstream sources as a Flux-managed resource +- keep the remaining e2e-specific resources in a final local manifests directory +- apply that manifests directory only after the operator CRDs are established -This avoids mixing one straightforward cleanup with a more open-ended refactor. +This keeps the setup cloud-native without forcing the entire Prometheus e2e definition into a chart values file. ## Completed step: replace `$(CS)/prometheus.installed` @@ -279,14 +281,16 @@ Out of scope: ### Recommended shape -Add Prometheus as another local Flux-managed service under `test/e2e/setup/flux-services/`. +Install the Prometheus Operator from upstream manifests through Flux and keep the last small set of e2e-specific +resources as a final local apply step. -That likely means: +That means: -- a `HelmRepository` for the Prometheus operator chart source -- a `HelmRelease` for the operator stack -- local manifests or Flux-managed resources for the e2e-specific Prometheus instance and `ServiceMonitor` -- the same shared `HelmRelease` readiness gate used by `$(CS)/services.ready` +- a Flux `GitRepository` pointing at the upstream Prometheus Operator repository +- a Flux `Kustomization` that reconciles the operator bundle into `prometheus-operator` +- a local `test/e2e/setup/manifests/` directory for the remaining resources +- a sequencing point in Make by waiting for the operator `Kustomization` and CRDs before applying that manifests + directory The key design goal is that `$(CS)/services.ready` should depend on one shared Flux services gate, not on a separate Prometheus install recipe. @@ -305,7 +309,8 @@ This step is done when: 1. Prometheus no longer has a dedicated install recipe in `Makefile`. 2. `prepare-e2e` still reaches a working shared Prometheus endpoint for the existing e2e flow. -3. The full shared-service bootstrap path is expressed through Flux-applied local manifests. +3. The operator bootstrap path is expressed through Flux-applied resources, with the final local manifests applied only + after the operator CRDs are ready. 4. `make test-e2e`, `make test-e2e-quickstart-manifest`, and `make test-e2e-quickstart-helm` still pass. ## Phase 3: Optional Flux-managed controller install mode @@ -332,11 +337,12 @@ Reasons: - easier local debugging - still exercises Flux controllers and reconciliation loops -In practice, Flux would reconcile local custom resources we apply directly: +In practice, Flux reconciles local custom resources we apply directly: - `HelmRepository` - `HelmRelease` -- later possibly `Kustomization` +- `GitRepository` +- `Kustomization` If this works well, a later phase can move the service definitions into a Git-backed source in Gitea for higher fidelity. @@ -367,13 +373,7 @@ test/e2e/setup/flux-services/ Optional helper location if needed: -```text -hack/e2e/ - install-flux.sh -``` - -The default preference should be to keep logic in Make small and use at most one focused helper script if readability -improves. +No helper scripts should be required for shared service installation once the Flux resources are in place. ## Proposed Makefile shape diff --git a/hack/e2e/ensure-prometheus-operator.sh b/hack/e2e/ensure-prometheus-operator.sh deleted file mode 100755 index 819cd1d52..000000000 --- a/hack/e2e/ensure-prometheus-operator.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -euo pipefail - -PROM_OPERATOR_VERSION="${PROM_OPERATOR_VERSION:-0.89.0}" -PROM_OPERATOR_NAMESPACE="${PROM_OPERATOR_NAMESPACE:-prometheus-operator}" -KUBE_CONTEXT="${KUBECONTEXT:-${CTX:-}}" - -if [[ -z "${KUBE_CONTEXT}" ]]; then - KUBE_CONTEXT="$(kubectl config current-context 2>/dev/null || true)" -fi - -if [[ -z "${KUBE_CONTEXT}" ]]; then - echo "❌ Kubernetes context is required (set KUBECONTEXT or CTX)" - exit 1 -fi - -is_fully_installed() { - kubectl --context "${KUBE_CONTEXT}" get deployment prometheus-operator -n "${PROM_OPERATOR_NAMESPACE}" >/dev/null 2>&1 && - kubectl --context "${KUBE_CONTEXT}" get crd prometheuses.monitoring.coreos.com >/dev/null 2>&1 && - kubectl --context "${KUBE_CONTEXT}" get crd servicemonitors.monitoring.coreos.com >/dev/null 2>&1 -} - -if is_fully_installed; then - echo "✅ Prometheus Operator already installed and configured in ${PROM_OPERATOR_NAMESPACE}" - exit 0 -fi - -echo "Installing Prometheus Operator v${PROM_OPERATOR_VERSION} in namespace ${PROM_OPERATOR_NAMESPACE}..." -kubectl --context "${KUBE_CONTEXT}" create namespace "${PROM_OPERATOR_NAMESPACE}" --dry-run=client -o yaml | \ - kubectl --context "${KUBE_CONTEXT}" apply -f - - -tmp_dir="$(mktemp -d)" -cleanup() { - rm -rf "${tmp_dir}" -} -trap cleanup EXIT - -cat > "${tmp_dir}/kustomization.yaml" < Date: Fri, 13 Mar 2026 11:44:46 +0000 Subject: [PATCH 7/8] chore: Preparing for merge, finishing docs and getting the flux stuff in a more clear folder name. Also ran updates. --- Makefile | 36 +- PROD_DEBUG_REPORT.md | 234 -------- docs/design/audit-webhook-redis-queue-plan.md | 272 +++++---- docs/design/flux-e2e-services-plan.md | 521 ------------------ go.mod | 80 +-- go.sum | 170 +++--- .../kustomization.yaml | 5 - .../namespaces/cert-manager.yaml | 0 .../namespaces/gitea-e2e.yaml | 0 .../namespaces/prometheus-operator.yaml | 0 .../namespaces/valkey-e2e.yaml | 0 .../releases/cert-manager.yaml | 0 .../releases/gitea.yaml | 0 .../releases/prometheus-operator.yaml | 0 .../releases/valkey.yaml | 0 .../repositories/gitea-charts.yaml | 0 .../repositories/jetstack.yaml | 0 .../repositories/prometheus-operator.yaml | 0 .../repositories/valkey.yaml | 0 .../values/gitea-values.yaml | 0 .../values/valkey-values.yaml | 0 21 files changed, 304 insertions(+), 1014 deletions(-) delete mode 100644 PROD_DEBUG_REPORT.md delete mode 100644 docs/design/flux-e2e-services-plan.md rename test/e2e/setup/{flux-services => flux}/kustomization.yaml (89%) rename test/e2e/setup/{flux-services => flux}/namespaces/cert-manager.yaml (100%) rename test/e2e/setup/{flux-services => flux}/namespaces/gitea-e2e.yaml (100%) rename test/e2e/setup/{flux-services => flux}/namespaces/prometheus-operator.yaml (100%) rename test/e2e/setup/{flux-services => flux}/namespaces/valkey-e2e.yaml (100%) rename test/e2e/setup/{flux-services => flux}/releases/cert-manager.yaml (100%) rename test/e2e/setup/{flux-services => flux}/releases/gitea.yaml (100%) rename test/e2e/setup/{flux-services => flux}/releases/prometheus-operator.yaml (100%) rename test/e2e/setup/{flux-services => flux}/releases/valkey.yaml (100%) rename test/e2e/setup/{flux-services => flux}/repositories/gitea-charts.yaml (100%) rename test/e2e/setup/{flux-services => flux}/repositories/jetstack.yaml (100%) rename test/e2e/setup/{flux-services => flux}/repositories/prometheus-operator.yaml (100%) rename test/e2e/setup/{flux-services => flux}/repositories/valkey.yaml (100%) rename test/e2e/setup/{flux-services => flux}/values/gitea-values.yaml (100%) rename test/e2e/setup/{flux-services => flux}/values/valkey-values.yaml (100%) diff --git a/Makefile b/Makefile index 899280d93..e32102ff7 100644 --- a/Makefile +++ b/Makefile @@ -292,8 +292,7 @@ VALKEY_PORT ?= 16379 FLUX_NAMESPACE ?= flux-system FLUX_WAIT_TIMEOUT ?= 300s FLUX_SERVICES_WAIT_TIMEOUT ?= 600s -FLUX_SERVICES_DIR ?= test/e2e/setup/flux-services -FLUX_SERVICES_SELECTOR ?= e2e.configbutler.io/stack=services +FLUX_SERVICES_DIR ?= test/e2e/setup/flux ENVTEST_STAMP = .stamps/envtest-$(ENVTEST_K8S_VERSION).ready @@ -322,8 +321,8 @@ $(CS)/ready: test/e2e/cluster/start-cluster.sh | $(CS) FLUX_SERVICES_INPUTS = $(shell find $(FLUX_SERVICES_DIR) -type f) LAST_STEP_MANIFESTS_DIR = test/e2e/setup/manifests LAST_STEP_MANIFESTS = $(shell find $(LAST_STEP_MANIFESTS_DIR) -type f -name '*.yaml') -FLUX_SERVICES_READY_RESOURCE_TYPES = helmreleases.helm.toolkit.fluxcd.io,kustomizations.kustomize.toolkit.fluxcd.io -SERVICES_READY_INPUTS = $(CS)/flux.installed $$(FLUX_SERVICES_INPUTS) $$(LAST_STEP_MANIFESTS) +FLUX_SETUP_READY_INPUTS = $(CS)/flux.installed $$(FLUX_SERVICES_INPUTS) +SERVICES_READY_INPUTS = $(CS)/flux-setup.ready $$(LAST_STEP_MANIFESTS) $(CS)/flux.installed: $(CS)/ready | $(CS) $(FLUX) install --context $(CTX) --namespace $(FLUX_NAMESPACE) kubectl --context $(CTX) -n $(FLUX_NAMESPACE) wait \ @@ -337,19 +336,28 @@ $(CS)/flux.installed: $(CS)/ready | $(CS) --timeout=$(FLUX_WAIT_TIMEOUT) $(FLUX) version --client > $@ -# Aggregate stamp for external E2E services required by tests. -$(CS)/services.ready: $(SERVICES_READY_INPUTS) | $(CS) - kubectl --context $(CTX) delete helmreleases.helm.toolkit.fluxcd.io/prometheus-shared-e2e -n prometheus-operator --ignore-not-found=true - kubectl --context $(CTX) delete helmrepositories.source.toolkit.fluxcd.io/prometheus-community -n flux-system --ignore-not-found=true - kubectl --context $(CTX) -n prometheus-operator delete prometheus.monitoring.coreos.com/prometheus-shared-e2e servicemonitor.monitoring.coreos.com/gitops-reverser --ignore-not-found=true - kubectl --context $(CTX) -n prometheus-operator delete deployment/prometheus-shared-e2e-operator statefulset/prometheus-prometheus-shared-e2e service/prometheus-shared-e2e-operator service/prometheus-shared-e2e-prometheus --ignore-not-found=true - kubectl --context $(CTX) -n prometheus-operator delete pod/prometheus-prometheus-shared-e2e-0 --ignore-not-found=true --force --grace-period=0 +# Aggregate stamp for Flux-managed external E2E services required by tests. +$(CS)/flux-setup.ready: $(FLUX_SETUP_READY_INPUTS) | $(CS) kubectl --context $(CTX) apply -k $(FLUX_SERVICES_DIR) - flux_ready_count="$$(kubectl --context $(CTX) get $(FLUX_SERVICES_READY_RESOURCE_TYPES) --all-namespaces -l $(FLUX_SERVICES_SELECTOR) --no-headers 2>/dev/null | wc -l | tr -d ' ')" + flux_ready_count=0; \ + for kind in \ + helmreleases.helm.toolkit.fluxcd.io \ + kustomizations.kustomize.toolkit.fluxcd.io \ + ; do \ + resources="$$(kubectl --context $(CTX) get $$kind --all-namespaces -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' 2>/dev/null)"; \ + [ -z "$$resources" ] && continue; \ + flux_ready_count="$$(($$flux_ready_count + $$(printf '%s\n' "$$resources" | sed '/^$$/d' | wc -l | tr -d ' ')))"; \ + printf '%s\n' "$$resources" | while read -r namespace name; do \ + [ -n "$$namespace" ] || continue; \ + kubectl --context $(CTX) -n "$$namespace" wait "$$kind/$$name" --for=condition=Ready --timeout=$(FLUX_SERVICES_WAIT_TIMEOUT); \ + done; \ + done; \ [ "$$flux_ready_count" -gt 0 ] || { echo "ERROR: no Flux-managed e2e ready-check resources found" >&2; exit 1; } - if ! kubectl --context $(CTX) wait $(FLUX_SERVICES_READY_RESOURCE_TYPES) --all-namespaces -l $(FLUX_SERVICES_SELECTOR) --for=condition=Ready --timeout=$(FLUX_SERVICES_WAIT_TIMEOUT); then kubectl --context $(CTX) get $(FLUX_SERVICES_READY_RESOURCE_TYPES) --all-namespaces -l $(FLUX_SERVICES_SELECTOR); exit 1; fi kubectl --context $(CTX) wait --for=condition=Established crd/prometheuses.monitoring.coreos.com crd/servicemonitors.monitoring.coreos.com --timeout=180s - + touch $@ + +# Aggregate stamp for external E2E services required by tests. +$(CS)/services.ready: $(SERVICES_READY_INPUTS) | $(CS) kubectl --context $(CTX) apply -R -f $(LAST_STEP_MANIFESTS_DIR) touch $@ diff --git a/PROD_DEBUG_REPORT.md b/PROD_DEBUG_REPORT.md deleted file mode 100644 index 624cb55ac..000000000 --- a/PROD_DEBUG_REPORT.md +++ /dev/null @@ -1,234 +0,0 @@ -# Production Debug Report: gitops-reverser - -Date: 2026-02-26 -Workspace: `/workspaces/gitops-reverser2` - -## Scope -- Investigate why configmap/secret changes in `gitops-reverser-example-audit-namespace` are not being tracked. -- Read-only investigation against cluster, except allowed restart of gitops-reverser and changes in `gitops-reverser-example-audit` namespace. - -## Findings Log - -### 1) Context and deployment health - -Command: -```bash -kubectl config current-context -kubectl -n gitops-reverser get all -helm list -A | rg gitops-reverser -``` - -Result: -- Current context: `k3s` -- Pod/Deployment/Service for gitops-reverser are healthy in namespace `gitops-reverser` -- Helm release: `gitops-reverser-gitops-reverser` chart `gitops-reverser-0.17.0+01bf7040249c` - ---- - -### 2) Config objects and watch scope - -Command: -```bash -kubectl -n gitops-reverser-example-audit get gittargets,watchrules -o yaml -kubectl explain watchrule.spec -kubectl get clusterwatchrules -A -o yaml -``` - -Result: -- `WatchRule only-configmaps` exists in `gitops-reverser-example-audit` -- Rule watches resources: `configmaps`, `secrets` -- `WatchRule` is namespace-scoped (per `kubectl explain watchrule.spec`) -- No `ClusterWatchRule` exists - -Interpretation: -- Current setup only tracks objects in namespace `gitops-reverser-example-audit`. - ---- - -### 3) Namespace mismatch check - -Command: -```bash -kubectl get ns | rg gitops-reverser-example -kubectl get ns gitops-reverser-example-audit-namespace -o yaml -``` - -Result: -- Existing namespace: `gitops-reverser-example-audit` -- Namespace `gitops-reverser-example-audit-namespace` does **not** exist - -Interpretation: -- If changes were made in `gitops-reverser-example-audit-namespace`, they cannot be tracked because the namespace does not exist. - ---- - -### 4) Metrics before test - -Command: -```bash -kubectl -n gitops-reverser exec deploy/gitops-reverser -- sh -c 'wget -qO- http://127.0.0.1:8080/metrics | grep "^gitopsreverser_" | sort' -``` - -Key values before controlled test: -- `gitopsreverser_events_received_total`: 6089 -- `gitopsreverser_events_processed_total`: 18 -- `gitopsreverser_commits_total`: 1 -- `gitopsreverser_objects_written_total`: 1 - ---- - -### 5) Controlled end-to-end test in `gitops-reverser-example-audit` - -Command: -```bash -ns="gitops-reverser-example-audit" -cm="codex-debug-cm-081911" -sec="codex-debug-secret-081911" - -kubectl -n "$ns" create configmap "$cm" --from-literal=foo=one -kubectl -n "$ns" create secret generic "$sec" --from-literal=foo=one -kubectl -n "$ns" create configmap "$cm" --from-literal=foo=two -o yaml --dry-run=client | kubectl apply -f - -kubectl -n "$ns" create secret generic "$sec" --from-literal=foo=two -o yaml --dry-run=client | kubectl apply -f - -kubectl -n "$ns" delete configmap "$cm" --wait=true -kubectl -n "$ns" delete secret "$sec" --wait=true -``` - -Log correlation command: -```bash -kubectl -n gitops-reverser logs deploy/gitops-reverser --since=5m | rg 'codex-debug-cm-081911|codex-debug-secret-081911|Created commit|All events pushed|writeEvents operation|Starting git commit and push' -``` - -Observed logs (highlights): -- `Starting git commit and push` with `eventCount: 6` -- `Created commit` for: - - `CREATE v1/configmaps/codex-debug-cm-081911` - - `CREATE v1/secrets/codex-debug-secret-081911` - - `UPDATE v1/configmaps/codex-debug-cm-081911` - - `UPDATE v1/secrets/codex-debug-secret-081911` - - `DELETE v1/configmaps/codex-debug-cm-081911` - - `DELETE v1/secrets/codex-debug-secret-081911` -- `All events pushed to remote` after each event - -Post-test metrics/status: - -Command: -```bash -kubectl -n gitops-reverser exec deploy/gitops-reverser -- sh -c 'wget -qO- http://127.0.0.1:8080/metrics | grep -E "^gitopsreverser_(events_received_total|events_processed_total|objects_written_total|commits_total|git_operations_total|git_commit_queue_size)" | sort' -kubectl -n gitops-reverser-example-audit get gittarget to-folder-live-cluster -o jsonpath='{.status.lastCommit}{"\n"}{.status.lastReconcileTime}{"\n"}{.status.snapshot.stats.created}{"\n"}{.status.snapshot.stats.deleted}{"\n"}' -``` - -Result: -- `events_processed_total`: 18 -> 24 -- `commits_total`: 1 -> 2 -- `git_operations_total`: 1 -> 7 -- `objects_written_total`: 1 -> 7 -- `GitTarget.status.lastCommit`: changed to `3e4cfd329e3b3e7638253bb3db3beb61ce363a0c` - -Interpretation: -- ConfigMap/Secret tracking is functioning in `gitops-reverser-example-audit`. - ---- - -### 6) Additional finding: startup seed race (likely non-blocking after startup) - -Command: -```bash -kubectl -n gitops-reverser logs deploy/gitops-reverser --since=20m | rg -i 'Failed to route event to GitTargetEventStream|dropping event' -``` - -Observed error: -- `Failed to route event to GitTargetEventStream - dropping event` -- `error: no GitTargetEventStream registered for gitops-reverser-example-audit/to-folder-live-cluster` -- Appears around startup/initial seed window. - -Interpretation: -- There is a race during initial seed listing where events can be dropped before the stream is registered. -- This did **not** prevent later live create/update/delete processing in this test. - ---- - -## Conclusion - -- The system is actively tracking ConfigMap and Secret changes in `gitops-reverser-example-audit` and pushing them to Git. -- Main issue found in this debugging session: namespace mismatch (`gitops-reverser-example-audit-namespace` does not exist; watch rule is namespace-scoped to `gitops-reverser-example-audit`). -- Secondary issue found: startup seed event-drop race exists in logs and may affect initial catch-up behavior at startup. - ---- - -### 7) Root cause for "committed then immediately removed" (`oeps3`) - -#### 7.1 Reproduction evidence from logs - -Command: -```bash -kubectl -n gitops-reverser logs deploy/gitops-reverser --since=90m | rg -n 'oeps3|Starting git commit and push|Deleted file from repository|Created commit' -``` - -Observed sequence: -- `08:21:02` -> `Created commit` `operation:"CREATE"` `resource:"v1/configmaps/oeps3"` -- `08:21:10` -> `Deleted file from repository` `file:"live-cluster/v1/configmaps/gitops-reverser-example-audit/oeps3.yaml"` -- `08:21:10` -> `Created commit` `operation:"DELETE"` `resource:"live-cluster/v1/configmaps/oeps3"` - -At the same time, object still exists in cluster: - -Command: -```bash -kubectl -n gitops-reverser-example-audit get cm oeps3 -o yaml -``` - -Result: -- `ConfigMap oeps3` exists (`creationTimestamp: 2026-02-26T08:20:55Z`). - -#### 7.2 Why this happens (code-path analysis) - -1) Repo-state reconciliation asks BranchWorker to list resources under GitTarget path: - -- [`internal/watch/event_router.go`](/workspaces/gitops-reverser2/internal/watch/event_router.go:151) - - `worker.ListResourcesInPath(gitTarget.Spec.Path)` - -2) BranchWorker walks `basePath = repoPath + path`, but computes relative path against **repo root**: - -- [`internal/git/branch_worker.go`](/workspaces/gitops-reverser2/internal/git/branch_worker.go:358) -- [`internal/git/branch_worker.go`](/workspaces/gitops-reverser2/internal/git/branch_worker.go:371) - - `relPath := filepath.Rel(repoPath, path)` - -This means files are parsed like: -- `live-cluster/v1/configmaps/gitops-reverser-example-audit/oeps3.yaml` - -3) Path parser interprets first segment as API group for 5-part paths: - -- [`internal/git/helpers.go`](/workspaces/gitops-reverser2/internal/git/helpers.go:97) - - For 5 parts, sets `group = parts[0]`. - -So parsed identifier becomes effectively: -- `group=live-cluster, version=v1, resource=configmaps, namespace=gitops-reverser-example-audit, name=oeps3` - -instead of core group (`group=""`). - -4) FolderReconciler compares cluster identifiers vs git identifiers and emits deletes for git-only items: - -- [`internal/reconcile/folder_reconciler.go`](/workspaces/gitops-reverser2/internal/reconcile/folder_reconciler.go:126) -- [`internal/reconcile/folder_reconciler.go`](/workspaces/gitops-reverser2/internal/reconcile/folder_reconciler.go:145) - -Because keys mismatch (`"/v1/..."` vs `"live-cluster/v1/..."`), it emits DELETE. - -5) Event stream behavior makes this asymmetrical: - -- [`internal/reconcile/git_target_event_stream.go`](/workspaces/gitops-reverser2/internal/reconcile/git_target_event_stream.go:161) - - Events with `Object=nil` are skipped **except DELETE**. - -So reconcile-created CREATE events are dropped, but reconcile-created DELETE events are processed, causing file removal commits. - -#### 7.3 Additional corroboration - -Command: -```bash -kubectl -n gitops-reverser exec deploy/gitops-reverser -- sh -c 'find /tmp/gitops-reverser-workers/gitops-reverser-example-audit/your-repo/test-gitops-reverser/repos/54d901857c780feacd94c0da35957f7a/live-cluster -type f | sort' -``` - -Result at capture time: -- `live-cluster/v1/configmaps/...` file for `oeps3` was absent in repo cache -- `oeps3` still existed in Kubernetes - -This confirms Git side deletion without cluster deletion. - diff --git a/docs/design/audit-webhook-redis-queue-plan.md b/docs/design/audit-webhook-redis-queue-plan.md index f813ec34a..9c2689ca5 100644 --- a/docs/design/audit-webhook-redis-queue-plan.md +++ b/docs/design/audit-webhook-redis-queue-plan.md @@ -1,46 +1,72 @@ -# Audit Webhook Only + Durable Redis Queue Plan +# Audit Webhook + Valkey Queue Plan -## Goal +## Current status -Move GitOps Reverser to a single ingestion path: +The audit webhook durable enqueue foundation is now in place. -1. Kubernetes Audit Webhook ingress only -2. Durable Redis-backed queue for all incoming audit events -3. Replayable, at-least-once delivery into Git processing +Implemented: -Note: this document uses **Redis** terminology for the queue contract. Your new Valkey instance is treated as a -Redis-compatible runtime. +1. Audit ingress is live on `/audit-webhook/{clusterID}`. +2. Accepted audit events can be enqueued into a Redis-compatible stream backed by Valkey. +3. Redis/Valkey enqueueing is wired into: + - `cmd/main.go` runtime flags + - `charts/gitops-reverser` Helm values + - `config/deployment.yaml` for the local manifest-based e2e install +4. Unit coverage exists for: + - audit handler enqueue success/failure behavior + - Redis-backed enqueue behavior + - audit server flag parsing and validation +5. E2E coverage exists for the audit-to-Valkey enqueue path. -## Why This Is the Next Step +Not implemented yet: -Current docs and code show: +1. A Redis consumer that feeds the existing Git processing pipeline +2. Consumer groups, ack/reclaim handling, or replay workers +3. Deduplication and writer-lock coordination +4. Audit-only ingestion cutover +5. Queue lag / PEL / end-to-end durability metrics -1. Audit ingress is already wired (`/audit-webhook/{clusterID}`) and stable. -2. Current queueing to Git workers is in-process/in-memory (`BranchWorker` channel), which is not durable. -3. Existing design direction already points to a durable queue layer and HA worker model. +## Current architecture -This plan closes the gap by making audit ingress durable first, then cutting over routing and scaling behavior. - -## Architecture Target +Today the flow is: ```text Kube API Audit Backend -> /audit-webhook/{clusterID} - -> Audit Handler (validate + normalize) - -> Redis Stream (durable enqueue) - -> Redis Consumer Group worker(s) - -> EventRouter / GitTargetEventStream / BranchWorker - -> Git commit/push + -> Audit Handler + -> Valkey/Redis stream enqueue (optional, feature-flagged) + -> stream persistence only ``` -## Data Contract +Important constraint: + +The stream is currently a durability buffer only. There is no in-process or external consumer reading from that stream +back into `EventRouter`, `GitTargetEventStream`, or `BranchWorker`. + +## Current implementation details + +### Audit handler enqueue integration + +The audit handler accepts an `AuditEventQueue` and enqueues each accepted audit event before returning success. + +Current behavior: + +1. If queueing is disabled, the handler behaves as before. +2. If queueing is enabled and enqueue succeeds, the request continues normally. +3. If queueing is enabled and enqueue fails, the handler returns `500` so the Kubernetes audit backend can retry. +4. Existing request-body guardrails and cluster path validation remain in place. + +This gives durable ingress semantics at the HTTP boundary. + +### Redis/Valkey stream producer + +The repository contains a concrete Redis-backed audit queue implementation under `internal/queue/`. -Use one Redis stream initially: +Current stream contract: 1. Stream: `gitopsreverser.audit.events.v1` -2. Consumer group: `gitopsreverser-writers` -3. Event fields: - - `event_id` (deterministic idempotency key) +2. Fields written today: + - `event_id` - `audit_id` - `cluster_id` - `verb` @@ -50,132 +76,142 @@ Use one Redis stream initially: - `name` - `user` - `stage_timestamp` - - `payload_json` (normalized event for downstream mapping) + - `payload_json` +3. `event_id` is a deterministic SHA-256 hash derived from cluster and audit event identity inputs. +4. Optional approximate trimming is supported through `maxLen`. + +Follow-up work should preserve this contract unless there is a deliberate migration plan. + +### Runtime and deployment wiring + +The queue can be enabled at runtime with dedicated audit Redis flags. + +Implemented controls: + +1. `--audit-redis-enabled` +2. `--audit-redis-addr` +3. `--audit-redis-username` +4. `--audit-redis-password` +5. `--audit-redis-db` +6. `--audit-redis-stream` +7. `--audit-redis-max-len` +8. `--audit-redis-tls` + +Deployment status: + +1. Helm chart support exists under `queue.redis.*`. +2. Helm defaults remain off, which keeps production behavior conservative. +3. `config/deployment.yaml` currently enables audit enqueueing against the e2e Valkey service. + +### E2E environment support + +The e2e cluster includes Valkey as a Flux-managed shared service, and the repository contains a dedicated e2e test +that verifies audit events land in the stream after Kubernetes object changes. -## Phased Plan +This gives confidence in: -### Phase 0: Feature-flagged foundation +1. kube-apiserver audit webhook delivery +2. controller audit endpoint handling +3. Redis-compatible stream writes +4. cluster wiring between the controller and Valkey -1. Add `internal/queue/` package with interfaces: - - `Producer.Enqueue(ctx, event) error` - - `Consumer.Run(ctx, handler) error` -2. Add queue mode config: - - `queue.mode=inmemory|redis` (default `inmemory` for safe rollout) - - `redis.addr`, `redis.username`, `redis.password`, `redis.tls`, `redis.stream`, `redis.group` -3. Add Helm values under `queue` and `redis` while keeping current behavior unchanged by default. +## Settings review -### Phase 1: Audit ingress -> Redis durable enqueue +The current Redis/Valkey settings are good enough for this slice: they are simple, understandable, and sufficient for +producer-only durable enqueueing. -1. Extend `internal/webhook/audit_handler.go` to publish every accepted audit event to Redis. -2. Keep current file dump behavior as optional debug path. -3. Return `5xx` on enqueue failure so Kubernetes audit backend retries. -4. Preserve request body size guardrails and path contract. +The main gaps to keep in mind before broader production rollout are: -Exit criteria: +1. `maxLen` defaults to unbounded retention, which is simple but risky if a consumer is missing or stalled. +2. The connection settings are intentionally minimal; there are no explicit timeout, retry, or pool controls yet. +3. TLS is currently only enabled/disabled; there is no CA or mTLS configuration surface. +4. Startup does not verify Redis reachability, so failures show up on enqueue rather than at boot. -1. All accepted audit events are visible in stream (`XADD` count increases). -2. Handler correctness tests pass for success/failure paths. +## Scope of this slice -### Phase 2: Redis consumer -> existing processing pipeline +This slice should be understood as: -1. Implement consumer worker runnable in `cmd/main.go` lifecycle. -2. Consume via consumer group (`XREADGROUP`) and `XACK` only after successful handoff to existing event pipeline. -3. On worker crash/stall, reclaim stuck entries via `XAUTOCLAIM`. -4. Start with one consumer process, then allow horizontal scaling. +1. audit webhook durable enqueue foundation +2. Valkey-backed buffering for accepted audit events +3. validated e2e plumbing for audit-to-stream delivery -Exit criteria: +This slice should not be described as: -1. No dropped events during pod restart test. -2. Pending Entries List (PEL) returns to steady state after failures. +1. complete durable queue processing +2. audit-only architecture cutover +3. HA-safe worker replay model -### Phase 3: Idempotency and ordering controls +## Remaining work -1. Add dedupe key in Redis: - - key: `gitopsreverser:dedupe:` - - set with TTL (for example 24h) before downstream processing. -2. Partition lock per `{provider,branch}` using `SET NX EX` lock keys to avoid concurrent writers. -3. Define lock TTL renewal while worker is active. +### Next step 1: Stream consumer into existing processing pipeline -Exit criteria: +Still needed: -1. Duplicate deliveries do not produce duplicate Git writes. -2. Parallel workers do not race on same branch writes. +1. consumer runnable lifecycle in `cmd/main.go` +2. consumer group creation and management +3. `XREADGROUP` read loop +4. handoff into the existing Git processing path +5. `XACK` only after safe downstream handoff +6. retry and reclaim behavior such as `XAUTOCLAIM` -### Phase 4: Audit-only ingestion cutover +Open design question: -1. Add runtime switch to disable validating-webhook correlation path. -2. Keep watch/reconcile pieces required for seed and rule-change reconciliation. -3. Remove dual-path assumptions from docs and operator config defaults. +We still need to decide the cleanest handoff boundary into the current `EventRouter` and branch-worker architecture so +we do not create a second partially overlapping pipeline. -Exit criteria: +### Next step 2: Idempotency and writer coordination -1. Ingestion path for ongoing events is audit-only in production profile. -2. Reconcile startup/rule-change still works. +Still needed: -### Phase 5: Performance and durability validation +1. dedupe key storage for `event_id` +2. TTL policy for dedupe entries +3. branch or target scoped writer locking +4. lock renewal and timeout rules -1. Add metrics: - - `audit_ingest_requests_total` - - `audit_enqueue_latency_seconds` - - `redis_stream_lag` - - `redis_pel_entries` - - `event_end_to_end_latency_seconds` - - `event_dropped_total` -2. Run repeatable load test profiles (low, nominal, burst). -3. Compare baseline (in-memory path) vs Redis mode on: - - p50/p95/p99 ingest latency - - sustained events/sec - - recovery time after Git outage or pod restart - - memory footprint under burst load +Risk if consumer work lands before this: -Exit criteria: +At-least-once delivery would likely produce duplicate downstream writes under retries or failover. -1. Throughput and reliability targets are met or tradeoff is explicitly accepted. -2. Performance report is committed under `docs/ci/`. +### Next step 3: Audit-only ingestion cutover -## Performance Expectations +Current state: -Likely impact: +1. the validating webhook correlation path still exists +2. watch/reconcile behavior remains part of the live ingestion model +3. nothing here removes those assumptions -1. Slightly higher per-request latency due to networked enqueue. -2. Significantly better durability and restart recovery. -3. Better burst handling when Git is slow/unavailable. +That means the system has not switched to audit-only operation yet. -Practical expectation is improved reliability under failure and load spikes, with small steady-state latency overhead. +### Next step 4: Metrics and durability validation -## Testing Strategy +What exists now: -1. Unit tests: - - Redis producer/consumer behavior - - enqueue failure handling in audit handler - - idempotency and lock behavior -2. Integration tests: - - consumer group ack/reclaim flows - - restart recovery with pending messages -3. E2E: - - send audit events, verify Git output and Redis ack behavior - - chaos case: stop worker, produce events, restart worker, verify replay +1. tests prove enqueue success and failure behavior +2. e2e proves stream writes happen -## Rollout Strategy +What still needs to be added: -1. Stage A: `queue.mode=inmemory` in production, `redis` in non-prod shadow. -2. Stage B: enable `redis` for one environment with one replica. -3. Stage C: scale replicas and validate lock/ordering behavior. -4. Stage D: set audit-only ingestion as default. +1. enqueue latency metrics +2. stream lag metrics +3. PEL metrics +4. end-to-end replay latency metrics +5. load or outage recovery reports -## Open Decisions +## Recommended next steps -1. One global stream vs stream-per-cluster/per-target -2. Dedupe TTL duration -3. Exact partition key for writer locks (`repo+branch` vs `target+branch`) -4. Dead-letter policy for poison events +Recommended follow-up order: -## First Implementation Slice (Recommended) +1. Merge this slice as the producer/enqueue foundation. +2. Keep the Flux/Valkey e2e support with it so the test story stays honest. +3. Add a focused follow-up PR for the consumer runnable and downstream handoff. +4. Add idempotency and locking before claiming at-least-once safe Git processing. +5. Revisit audit-only cutover language in README and architecture docs only after the consumer path is real. -Deliver first: +## Exit criteria for this slice -1. Redis producer in audit handler (Phase 1) -2. Minimal consumer runnable with `XREADGROUP` + `XACK` (Phase 2) -3. Basic lag/PEL metrics (part of Phase 5 instrumentation) +This slice is complete when all of the following are true: -This gives durable ingress quickly and lets you measure performance early before deeper refactors. +1. Audit events are durably appended to Valkey/Redis when queueing is enabled. +2. Enqueue failures surface as `500` responses. +3. The stream contract is documented and stable enough for a future consumer. +4. Unit and e2e coverage keep the enqueue path from regressing. diff --git a/docs/design/flux-e2e-services-plan.md b/docs/design/flux-e2e-services-plan.md deleted file mode 100644 index 61f9df5ae..000000000 --- a/docs/design/flux-e2e-services-plan.md +++ /dev/null @@ -1,521 +0,0 @@ -# Flux-Managed E2E Services Plan - -## Status update - -The first Flux migration is now in place. - -Completed: - -- Flux controllers are bootstrapped from the Makefile -- shared e2e services are applied from `test/e2e/setup/flux-services/` -- `cert-manager`, `gitea`, and `valkey` are reconciled through Flux `HelmRelease` resources -- the Prometheus Operator is reconciled through a Flux `GitRepository` + `Kustomization` -- a final local manifests step applies the remaining e2e-specific resources after the operator CRDs are ready -- `$(CS)/services.ready` is now the single shared-services readiness gate - -Still intentionally outside that migration: - -- the controller install itself -- Git-backed Flux sources - -## Goal - -Reduce e2e setup complexity by making Flux manage the shared support services used by the test cluster. - -Initial scope: - -- `cert-manager` -- `gitea` -- `valkey` - -Out of scope for the first pass: - -- moving the gitops-reverser controller install itself under Flux -- requiring a Git-backed Flux source for e2e bootstrapping -- changing the existing quickstart install-mode coverage (`helm`, `plain-manifests-file`, `config-dir`) - -## Why this direction - -The current e2e bootstrap mixes several orchestration styles: - -- imperative `kubectl apply` for `cert-manager` -- imperative `helm upgrade --install` for `gitea` -- imperative `helm upgrade --install` for `valkey` -- imperative shell setup for Prometheus operator bootstrap -- separate install-mode logic for the system under test - -That works, but it spreads lifecycle logic across the Makefile and shell scripts. - -The cleanest first step is to make Flux own the shared service layer while leaving the system-under-test install -flow unchanged. That gives us: - -- fewer imperative Helm steps in the Makefile -- a single place for service definitions -- explicit dependency ordering via Flux `dependsOn` -- a more GitOps-shaped e2e environment without introducing circular bootstrap problems - -## Recommendation - -Create a single Flux-applied services bundle under: - -`test/e2e/setup/flux-services/` - -That bundle should contain: - -- Flux services manifests -- `HelmRepository` resources for upstream charts -- `HelmRelease` resources for `cert-manager`, `gitea`, and `valkey` -- namespaces for those services -- any values/config needed by those Helm releases - -The Makefile should always apply this bundle as part of cluster services preparation. - -Flux should not initially reconcile from a Git repository. -Instead, the e2e setup should `kubectl apply` the Flux custom resources from the local working tree and then wait for -Flux to reconcile them in-cluster. - -This keeps bootstrap simple while still using Flux as the control plane for service installs. - -## Current seams in the Makefile - -Today the shared services are built around these cluster-scoped stamps: - -- `$(CS)/cert-manager.installed` -- `$(CS)/gitea.installed` -- `$(CS)/valkey.installed` -- `$(CS)/prometheus.installed` -- `$(CS)/services.ready` - -The imperative Helm/install logic currently lives in: - -- `$(CS)/cert-manager.installed` -- `$(CS)/gitea.installed` -- `$(CS)/valkey.installed` - -The system under test depends on `$(CS)/services.ready`, and the Go e2e suite always triggers that path through -`prepare-e2e`. - -That means the Flux migration should preserve the same high-level contract: - -`prepare-e2e` -> shared services ready -> controller install -> image override -> tests - -## Target architecture - -### 1. Shared cluster bootstrap - -Keep a small amount of imperative bootstrap in Make: - -- create/reuse the k3d cluster -- install Flux controllers -- apply the local Flux services bundle -- wait for all Flux-managed `HelmRelease` objects to become Ready - -This replaces direct Helm commands in Make, but it does not try to eliminate all Make orchestration. - -### 2. Flux-owned services bundle - -Create a local manifest tree like: - -```text -test/e2e/setup/flux-services/ - kustomization.yaml - namespaces/ - cert-manager.yaml - gitea-e2e.yaml - valkey-e2e.yaml - repositories/ - jetstack.yaml - gitea.yaml - valkey.yaml - releases/ - cert-manager.yaml - gitea.yaml - valkey.yaml - values/ - gitea-values.yaml - valkey-values.yaml -``` - -Recommended behavior: - -- `cert-manager` installs first -- `gitea` and `valkey` depend only on their repositories and namespaces -- if webhook/certificate timing becomes relevant later, additional `dependsOn` can be added cleanly - -### 3. Minimal Makefile contract - -Everything managed by Flux should disappear as a service-specific Make target. - -The target shape should become: - -- `$(CS)/flux.installed` -- `$(CS)/services.ready` - -That means: - -- no `$(CS)/cert-manager.ready` -- no `$(CS)/gitea.ready` -- no `$(CS)/valkey.ready` -- no service-specific install stamps for Flux-owned services - -The Makefile should not know how each service is installed anymore. -It should only know: - -1. Flux is installed. -2. The local Flux bundle was applied. -3. All Flux-owned `HelmRelease` objects are Ready. - -That is the cleanest simplification boundary. - -## Proposed implementation phases - -## Phase 1: Flux for cert-manager, Gitea, and Valkey - -### Desired outcome - -Replace: - -- direct `kubectl apply` of remote cert-manager manifest -- direct Helm install of Gitea -- direct Helm install of Valkey - -With: - -- Flux controllers installed once per e2e cluster context -- local Flux resources applied from `test/e2e/setup/flux-services/` -- one readiness gate based on Flux-managed `HelmRelease` objects - -### Concrete changes - -1. Add a new local Flux services directory. -2. Define service namespaces as plain manifests. -3. Define `HelmRepository` resources for: - - Jetstack - - Gitea charts - - Valkey charts -4. Define `HelmRelease` resources for: - - `cert-manager` - - `gitea` - - `valkey` -5. Reference existing e2e values for Gitea and Valkey, or move those values into the new directory if that makes the - bundle easier to understand. -6. Add `dependsOn` only where it improves determinism; do not over-model dependencies. -7. Replace the old `$(CS)/cert-manager.installed`, `$(CS)/gitea.installed`, and `$(CS)/valkey.installed` recipes with: - - Flux bootstrap - - apply local bundle - - a single `services.ready` wait - -### Readiness strategy - -Use one Make-level wait: - -1. Query all `HelmRelease` objects in the Flux services bundle. -2. Fail unless every one reports `Ready=True`. - -Recommended implementation shape: - -- label all Flux-owned e2e `HelmRelease` objects consistently -- use one `kubectl`-driven readiness check against that label set - -Example direction: - -```bash -kubectl --context "${CTX}" get helmreleases.helm.toolkit.fluxcd.io -A \ - -l e2e.configbutler.io/stack=services -``` - -This keeps the Makefile simple while still leaving detailed failure information in the `HelmRelease` status when -something goes wrong. - -If a specific workload later proves flaky even when the `HelmRelease` is Ready, we can add a focused workload-level -wait at that time. It should not be the default shape of the first implementation. - -### Makefile simplification target - -The Makefile should stop containing: - -- chart repo add/update commands -- per-service `helm upgrade --install` logic -- service-specific inline Helm arguments - -The Makefile should only contain: - -- bootstrap Flux -- apply the local Flux services bundle -- run one readiness check for all Flux-managed `HelmRelease` objects - -That is the main line-count and complexity win. - -## Phase 2: Prometheus via Flux-managed operator - -Prometheus remains slightly different from the other shared services because it depends on CRDs and an operator-managed -control loop. - -Recommendation: - -- install the Prometheus Operator from upstream sources as a Flux-managed resource -- keep the remaining e2e-specific resources in a final local manifests directory -- apply that manifests directory only after the operator CRDs are established - -This keeps the setup cloud-native without forcing the entire Prometheus e2e definition into a chart values file. - -## Completed step: replace `$(CS)/prometheus.installed` - -This increment replaced the old Prometheus-specific stamp target in the Makefile. - -### Scope - -In scope: - -- remove the special-case `$(CS)/prometheus.installed` install path from the Makefile -- move Prometheus bootstrap behind the same Flux-managed service pattern already used for `cert-manager`, `gitea`, and - `valkey` -- preserve the existing shared-service contract of `$(CS)/services.ready` - -Out of scope: - -- changing Prometheus scrape behavior -- redesigning test port-forwarding -- migrating the controller install under Flux -- introducing a Git-backed Flux source - -### Recommended shape - -Install the Prometheus Operator from upstream manifests through Flux and keep the last small set of e2e-specific -resources as a final local apply step. - -That means: - -- a Flux `GitRepository` pointing at the upstream Prometheus Operator repository -- a Flux `Kustomization` that reconciles the operator bundle into `prometheus-operator` -- a local `test/e2e/setup/manifests/` directory for the remaining resources -- a sequencing point in Make by waiting for the operator `Kustomization` and CRDs before applying that manifests - directory - -The key design goal is that `$(CS)/services.ready` should depend on one shared Flux services gate, not on a separate -Prometheus install recipe. - -### Makefile target end state - -After this step, the preferred shape is: - -- keep `$(CS)/flux.installed` -- remove `$(CS)/prometheus.installed` -- keep `$(CS)/services.ready` as the single shared-services readiness target - -### Acceptance criteria for this step - -This step is done when: - -1. Prometheus no longer has a dedicated install recipe in `Makefile`. -2. `prepare-e2e` still reaches a working shared Prometheus endpoint for the existing e2e flow. -3. The operator bootstrap path is expressed through Flux-applied resources, with the final local manifests applied only - after the operator CRDs are ready. -4. `make test-e2e`, `make test-e2e-quickstart-manifest`, and `make test-e2e-quickstart-helm` still pass. - -## Phase 3: Optional Flux-managed controller install mode - -Only after the service layer is stable, consider a new install mode for the controller itself. - -Possible future mode: - -- `INSTALL_MODE=flux-helm` - -But this should not be part of the first implementation because the current e2e flow patches the controller Deployment -image after install. A Flux-managed Deployment would likely reconcile that change away unless the image override is -designed into the Flux resources. - -## Why not use a GitRepository source first - -Using Flux without a Git source is the right first step here. - -Reasons: - -- simpler bootstrap -- fewer moving parts -- no circular dependency on Gitea to install Gitea -- easier local debugging -- still exercises Flux controllers and reconciliation loops - -In practice, Flux reconciles local custom resources we apply directly: - -- `HelmRepository` -- `HelmRelease` -- `GitRepository` -- `Kustomization` - -If this works well, a later phase can move the service definitions into a Git-backed source in Gitea for higher -fidelity. - -## Proposed file layout - -Recommended initial layout: - -```text -test/e2e/setup/flux-services/ - kustomization.yaml - namespaces/ - cert-manager.yaml - gitea-e2e.yaml - valkey-e2e.yaml - repositories/ - jetstack.yaml - gitea-charts.yaml - valkey.yaml - releases/ - cert-manager.yaml - gitea.yaml - valkey.yaml - values/ - gitea-values.yaml - valkey-values.yaml -``` - -Optional helper location if needed: - -No helper scripts should be required for shared service installation once the Flux resources are in place. - -## Proposed Makefile shape - -### New stamps - -- `$(CS)/flux.installed` -- `$(CS)/services.ready` - -### Updated aggregate - -`$(CS)/services.ready` should remain the single readiness stamp for shared e2e services. - -### What should be removed - -Remove or replace: - -- `GITEA_HELM_REPO_*` Make variables -- `VALKEY_HELM_REPO_*` Make variables -- direct Helm install recipes for Gitea and Valkey -- remote cert-manager manifest apply recipe -- per-service readiness stamps for Flux-owned services - -The chart versions should live with the Flux `HelmRelease` resources instead of the Makefile. - -## Flux bootstrap recommendation - -Flux itself should be bootstrapped separately from the services bundle. - -Recommended shape: - -1. Keep one stamp for Flux controller installation: - - `$(CS)/flux.installed` -2. Install Flux controllers into the cluster with the Flux CLI: - - `flux install` -3. Wait for Flux controllers in `flux-system` to be Available. -4. After that, apply `test/e2e/setup/flux-services/`. - -Why separate it from the services bundle: - -- the bundle contains Flux custom resources, so the CRDs/controllers must already exist -- Flux installation changes much less often than the service definitions -- it keeps the failure mode obvious: either Flux itself failed, or a Flux-managed service failed - -Alternative: - -- commit raw Flux install manifests into the repo and apply them with `kubectl` - -That works too, but the Flux CLI is the cleaner bootstrap tool for local e2e because it gives a short, supported way to -install the controllers without carrying a large generated manifest in-tree. - -## Flux CLI in the devcontainer - -Yes, the Flux CLI should be installed in the devcontainer if we adopt this plan. - -Reasoning: - -- the Makefile will depend on it for `$(CS)/flux.installed` -- local e2e and CI/devcontainer behavior should match -- Flux bootstrap is easier and less noisy through `flux install` than by vendoring install YAML - -Suggested approach: - -- add `FLUX_VERSION` alongside the other tool versions in `.devcontainer/Dockerfile` -- install the `flux` binary in the same style as `kubectl`, `helm`, and `k3d` - -What I would not do: - -- require developers to install Flux manually outside the container -- make the e2e bootstrap fetch and pipe an ad-hoc shell installer every run - -If we later decide to avoid the Flux CLI entirely, we can switch to checked-in install manifests and remove the binary. -But for the initial implementation, shipping the CLI in the devcontainer is the simpler and cleaner choice. - -## Cleanup model - -Cleanup should stay simple. - -Recommended approach: - -- keep Flux controllers installed for the cluster context once created -- treat Flux-managed services as shared cluster-scoped e2e prerequisites -- continue cleaning only per-test install namespaces during normal install cleanup -- add a dedicated target if full service teardown is needed - -This matches the current stamp philosophy of reusing expensive cluster setup work. - -## Risks and mitigations - -### Risk: readiness becomes less obvious - -Mitigation: - -- use one labeled `HelmRelease` readiness check -- inspect `HelmRelease` status and events when debugging failures - -### Risk: too much abstraction hides failures - -Mitigation: - -- keep manifests local and readable -- do not generate this bundle dynamically -- prefer explicit YAML over large shell wrappers - -### Risk: cert-manager ordering issues - -Mitigation: - -- install cert-manager through its own `HelmRelease` -- use `dependsOn` only where useful -- keep workload waits explicit in Make - -### Risk: trying to migrate too much at once - -Mitigation: - -- do cert-manager, Gitea, and Valkey first -- postpone Prometheus and controller install-mode changes - -## Success criteria - -Phase 1 is successful when: - -1. The Makefile has materially fewer service-install lines and less inline Helm logic. -2. `prepare-e2e` still remains the single entrypoint used by the Go suite. -3. Shared services are reconciled by Flux from local manifests. -4. `make test-e2e`, `make test-e2e-quickstart-manifest`, and `make test-e2e-quickstart-helm` still pass. -5. Local debugging remains straightforward: inspect Flux `HelmRelease` objects first, then workloads if needed. - -## Suggested execution order - -1. Add Flux bootstrap target and stamps. -2. Add `test/e2e/setup/flux-services/` with `HelmRepository` and `HelmRelease` YAMLs. -3. Migrate `cert-manager` into Flux. -4. Migrate `gitea` into Flux. -5. Migrate `valkey` into Flux. -6. Rewire `$(CS)/services.ready` to be the single shared-services readiness target. -7. Remove old imperative Helm/install logic from the Makefile. -8. Run full validation. - -## Notes for implementation - -- Keep the implementation aggressive about deleting now-redundant Makefile code. -- Prefer one shared services folder over per-service bootstrap patterns. -- Prefer stable, checked-in YAML over dynamically rendered e2e service manifests. -- Keep the controller install path separate until image override semantics are redesigned for Flux ownership. diff --git a/go.mod b/go.mod index 01234b010..1d554e50f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.1 require ( filippo.io/age v1.3.1 + github.com/alicebob/miniredis/v2 v2.37.0 github.com/cespare/xxhash/v2 v2.3.0 github.com/go-git/go-billy/v5 v5.8.0 github.com/go-git/go-git/v5 v5.17.0 @@ -12,13 +13,14 @@ require ( github.com/onsi/gomega v1.39.1 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/common v0.67.5 + github.com/redis/go-redis/v9 v9.18.0 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.42.0 go.opentelemetry.io/otel/exporters/prometheus v0.64.0 go.opentelemetry.io/otel/metric v1.42.0 go.opentelemetry.io/otel/sdk/metric v1.42.0 go.uber.org/zap v1.27.1 - golang.org/x/crypto v0.48.0 + golang.org/x/crypto v0.49.0 k8s.io/api v0.35.2 k8s.io/apiextensions-apiserver v0.35.2 k8s.io/apimachinery v0.35.2 @@ -34,8 +36,7 @@ require ( filippo.io/hpke v0.4.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/ProtonMail/go-crypto v1.3.0 // indirect - github.com/alicebob/miniredis/v2 v2.37.0 // indirect + github.com/ProtonMail/go-crypto v1.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect @@ -53,27 +54,27 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.22.4 // indirect - github.com/go-openapi/jsonreference v0.21.4 // indirect - github.com/go-openapi/swag v0.25.4 // indirect - github.com/go-openapi/swag/cmdutils v0.25.4 // indirect - github.com/go-openapi/swag/conv v0.25.4 // indirect - github.com/go-openapi/swag/fileutils v0.25.4 // indirect - github.com/go-openapi/swag/jsonname v0.25.4 // indirect - github.com/go-openapi/swag/jsonutils v0.25.4 // indirect - github.com/go-openapi/swag/loading v0.25.4 // indirect - github.com/go-openapi/swag/mangling v0.25.4 // indirect - github.com/go-openapi/swag/netutils v0.25.4 // indirect - github.com/go-openapi/swag/stringutils v0.25.4 // indirect - github.com/go-openapi/swag/typeutils v0.25.4 // indirect - github.com/go-openapi/swag/yamlutils v0.25.4 // indirect + github.com/go-openapi/jsonpointer v0.22.5 // indirect + github.com/go-openapi/jsonreference v0.21.5 // indirect + github.com/go-openapi/swag v0.25.5 // indirect + github.com/go-openapi/swag/cmdutils v0.25.5 // indirect + github.com/go-openapi/swag/conv v0.25.5 // indirect + github.com/go-openapi/swag/fileutils v0.25.5 // indirect + github.com/go-openapi/swag/jsonname v0.25.5 // indirect + github.com/go-openapi/swag/jsonutils v0.25.5 // indirect + github.com/go-openapi/swag/loading v0.25.5 // indirect + github.com/go-openapi/swag/mangling v0.25.5 // indirect + github.com/go-openapi/swag/netutils v0.25.5 // indirect + github.com/go-openapi/swag/stringutils v0.25.5 // indirect + github.com/go-openapi/swag/typeutils v0.25.5 // indirect + github.com/go-openapi/swag/yamlutils v0.25.5 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/cel-go v0.27.0 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20260202012954-cb029daf43ef // indirect + github.com/google/pprof v0.0.0-20260302011040-a15ffb7f9dcc // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -88,8 +89,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect - github.com/prometheus/procfs v0.19.2 // indirect - github.com/redis/go-redis/v9 v9.18.0 // indirect + github.com/prometheus/procfs v0.20.1 // indirect github.com/sergi/go-diff v1.4.0 // indirect github.com/skeema/knownhosts v1.3.2 // indirect github.com/spf13/cobra v1.10.2 // indirect @@ -98,38 +98,38 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect go.opentelemetry.io/otel/sdk v1.42.0 // indirect go.opentelemetry.io/otel/trace v1.42.0 // indirect - go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect - golang.org/x/mod v0.33.0 // indirect - golang.org/x/net v0.50.0 // indirect - golang.org/x/oauth2 v0.35.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/term v0.40.0 // indirect - golang.org/x/text v0.34.0 // indirect - golang.org/x/time v0.14.0 // indirect - golang.org/x/tools v0.42.0 // indirect + golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 // indirect + golang.org/x/mod v0.34.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/term v0.41.0 // indirect + golang.org/x/text v0.35.0 // indirect + golang.org/x/time v0.15.0 // indirect + golang.org/x/tools v0.43.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260217215200-42d3e9bedb6d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d // indirect - google.golang.org/grpc v1.79.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260311181403-84a4fc48630c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c // indirect + google.golang.org/grpc v1.79.2 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/component-base v0.35.2 // indirect - k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260304202019-5b3e3fdb0acf // indirect k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect diff --git a/go.sum b/go.sum index 400cee36b..49f9d79c9 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,8 @@ github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lpr github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= -github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= +github.com/ProtonMail/go-crypto v1.4.0 h1:Zq/pbM3F5DFgJiMouxEdSVY44MVoQNEKp5d5QxIQceQ= +github.com/ProtonMail/go-crypto v1.4.0/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= github.com/alicebob/miniredis/v2 v2.37.0 h1:RheObYW32G1aiJIj81XVt78ZHJpHonHLHW7OLIshq68= github.com/alicebob/miniredis/v2 v2.37.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= @@ -27,6 +27,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -80,40 +84,40 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.22.4 h1:dZtK82WlNpVLDW2jlA1YCiVJFVqkED1MegOUy9kR5T4= -github.com/go-openapi/jsonpointer v0.22.4/go.mod h1:elX9+UgznpFhgBuaMQ7iu4lvvX1nvNsesQ3oxmYTw80= -github.com/go-openapi/jsonreference v0.21.4 h1:24qaE2y9bx/q3uRK/qN+TDwbok1NhbSmGjjySRCHtC8= -github.com/go-openapi/jsonreference v0.21.4/go.mod h1:rIENPTjDbLpzQmQWCj5kKj3ZlmEh+EFVbz3RTUh30/4= -github.com/go-openapi/swag v0.25.4 h1:OyUPUFYDPDBMkqyxOTkqDYFnrhuhi9NR6QVUvIochMU= -github.com/go-openapi/swag v0.25.4/go.mod h1:zNfJ9WZABGHCFg2RnY0S4IOkAcVTzJ6z2Bi+Q4i6qFQ= -github.com/go-openapi/swag/cmdutils v0.25.4 h1:8rYhB5n6WawR192/BfUu2iVlxqVR9aRgGJP6WaBoW+4= -github.com/go-openapi/swag/cmdutils v0.25.4/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= -github.com/go-openapi/swag/conv v0.25.4 h1:/Dd7p0LZXczgUcC/Ikm1+YqVzkEeCc9LnOWjfkpkfe4= -github.com/go-openapi/swag/conv v0.25.4/go.mod h1:3LXfie/lwoAv0NHoEuY1hjoFAYkvlqI/Bn5EQDD3PPU= -github.com/go-openapi/swag/fileutils v0.25.4 h1:2oI0XNW5y6UWZTC7vAxC8hmsK/tOkWXHJQH4lKjqw+Y= -github.com/go-openapi/swag/fileutils v0.25.4/go.mod h1:cdOT/PKbwcysVQ9Tpr0q20lQKH7MGhOEb6EwmHOirUk= -github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ9869Vye1bNHI= -github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= -github.com/go-openapi/swag/jsonutils v0.25.4 h1:VSchfbGhD4UTf4vCdR2F4TLBdLwHyUDTd1/q4i+jGZA= -github.com/go-openapi/swag/jsonutils v0.25.4/go.mod h1:7OYGXpvVFPn4PpaSdPHJBtF0iGnbEaTk8AvBkoWnaAY= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4 h1:IACsSvBhiNJwlDix7wq39SS2Fh7lUOCJRmx/4SN4sVo= -github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.4/go.mod h1:Mt0Ost9l3cUzVv4OEZG+WSeoHwjWLnarzMePNDAOBiM= -github.com/go-openapi/swag/loading v0.25.4 h1:jN4MvLj0X6yhCDduRsxDDw1aHe+ZWoLjW+9ZQWIKn2s= -github.com/go-openapi/swag/loading v0.25.4/go.mod h1:rpUM1ZiyEP9+mNLIQUdMiD7dCETXvkkC30z53i+ftTE= -github.com/go-openapi/swag/mangling v0.25.4 h1:2b9kBJk9JvPgxr36V23FxJLdwBrpijI26Bx5JH4Hp48= -github.com/go-openapi/swag/mangling v0.25.4/go.mod h1:6dxwu6QyORHpIIApsdZgb6wBk/DPU15MdyYj/ikn0Hg= -github.com/go-openapi/swag/netutils v0.25.4 h1:Gqe6K71bGRb3ZQLusdI8p/y1KLgV4M/k+/HzVSqT8H0= -github.com/go-openapi/swag/netutils v0.25.4/go.mod h1:m2W8dtdaoX7oj9rEttLyTeEFFEBvnAx9qHd5nJEBzYg= -github.com/go-openapi/swag/stringutils v0.25.4 h1:O6dU1Rd8bej4HPA3/CLPciNBBDwZj9HiEpdVsb8B5A8= -github.com/go-openapi/swag/stringutils v0.25.4/go.mod h1:GTsRvhJW5xM5gkgiFe0fV3PUlFm0dr8vki6/VSRaZK0= -github.com/go-openapi/swag/typeutils v0.25.4 h1:1/fbZOUN472NTc39zpa+YGHn3jzHWhv42wAJSN91wRw= -github.com/go-openapi/swag/typeutils v0.25.4/go.mod h1:Ou7g//Wx8tTLS9vG0UmzfCsjZjKhpjxayRKTHXf2pTE= -github.com/go-openapi/swag/yamlutils v0.25.4 h1:6jdaeSItEUb7ioS9lFoCZ65Cne1/RZtPBZ9A56h92Sw= -github.com/go-openapi/swag/yamlutils v0.25.4/go.mod h1:MNzq1ulQu+yd8Kl7wPOut/YHAAU/H6hL91fF+E2RFwc= -github.com/go-openapi/testify/enable/yaml/v2 v2.0.2 h1:0+Y41Pz1NkbTHz8NngxTuAXxEodtNSI1WG1c/m5Akw4= -github.com/go-openapi/testify/enable/yaml/v2 v2.0.2/go.mod h1:kme83333GCtJQHXQ8UKX3IBZu6z8T5Dvy5+CW3NLUUg= -github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= -github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= +github.com/go-openapi/jsonpointer v0.22.5 h1:8on/0Yp4uTb9f4XvTrM2+1CPrV05QPZXu+rvu2o9jcA= +github.com/go-openapi/jsonpointer v0.22.5/go.mod h1:gyUR3sCvGSWchA2sUBJGluYMbe1zazrYWIkWPjjMUY0= +github.com/go-openapi/jsonreference v0.21.5 h1:6uCGVXU/aNF13AQNggxfysJ+5ZcU4nEAe+pJyVWRdiE= +github.com/go-openapi/jsonreference v0.21.5/go.mod h1:u25Bw85sX4E2jzFodh1FOKMTZLcfifd1Q+iKKOUxExw= +github.com/go-openapi/swag v0.25.5 h1:pNkwbUEeGwMtcgxDr+2GBPAk4kT+kJ+AaB+TMKAg+TU= +github.com/go-openapi/swag v0.25.5/go.mod h1:B3RT6l8q7X803JRxa2e59tHOiZlX1t8viplOcs9CwTA= +github.com/go-openapi/swag/cmdutils v0.25.5 h1:yh5hHrpgsw4NwM9KAEtaDTXILYzdXh/I8Whhx9hKj7c= +github.com/go-openapi/swag/cmdutils v0.25.5/go.mod h1:pdae/AFo6WxLl5L0rq87eRzVPm/XRHM3MoYgRMvG4A0= +github.com/go-openapi/swag/conv v0.25.5 h1:wAXBYEXJjoKwE5+vc9YHhpQOFj2JYBMF2DUi+tGu97g= +github.com/go-openapi/swag/conv v0.25.5/go.mod h1:CuJ1eWvh1c4ORKx7unQnFGyvBbNlRKbnRyAvDvzWA4k= +github.com/go-openapi/swag/fileutils v0.25.5 h1:B6JTdOcs2c0dBIs9HnkyTW+5gC+8NIhVBUwERkFhMWk= +github.com/go-openapi/swag/fileutils v0.25.5/go.mod h1:V3cT9UdMQIaH4WiTrUc9EPtVA4txS0TOmRURmhGF4kc= +github.com/go-openapi/swag/jsonname v0.25.5 h1:8p150i44rv/Drip4vWI3kGi9+4W9TdI3US3uUYSFhSo= +github.com/go-openapi/swag/jsonname v0.25.5/go.mod h1:jNqqikyiAK56uS7n8sLkdaNY/uq6+D2m2LANat09pKU= +github.com/go-openapi/swag/jsonutils v0.25.5 h1:XUZF8awQr75MXeC+/iaw5usY/iM7nXPDwdG3Jbl9vYo= +github.com/go-openapi/swag/jsonutils v0.25.5/go.mod h1:48FXUaz8YsDAA9s5AnaUvAmry1UcLcNVWUjY42XkrN4= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5 h1:SX6sE4FrGb4sEnnxbFL/25yZBb5Hcg1inLeErd86Y1U= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.5/go.mod h1:/2KvOTrKWjVA5Xli3DZWdMCZDzz3uV/T7bXwrKWPquo= +github.com/go-openapi/swag/loading v0.25.5 h1:odQ/umlIZ1ZVRteI6ckSrvP6e2w9UTF5qgNdemJHjuU= +github.com/go-openapi/swag/loading v0.25.5/go.mod h1:I8A8RaaQ4DApxhPSWLNYWh9NvmX2YKMoB9nwvv6oW6g= +github.com/go-openapi/swag/mangling v0.25.5 h1:hyrnvbQRS7vKePQPHHDso+k6CGn5ZBs5232UqWZmJZw= +github.com/go-openapi/swag/mangling v0.25.5/go.mod h1:6hadXM/o312N/h98RwByLg088U61TPGiltQn71Iw0NY= +github.com/go-openapi/swag/netutils v0.25.5 h1:LZq2Xc2QI8+7838elRAaPCeqJnHODfSyOa7ZGfxDKlU= +github.com/go-openapi/swag/netutils v0.25.5/go.mod h1:lHbtmj4m57APG/8H7ZcMMSWzNqIQcu0RFiXrPUara14= +github.com/go-openapi/swag/stringutils v0.25.5 h1:NVkoDOA8YBgtAR/zvCx5rhJKtZF3IzXcDdwOsYzrB6M= +github.com/go-openapi/swag/stringutils v0.25.5/go.mod h1:PKK8EZdu4QJq8iezt17HM8RXnLAzY7gW0O1KKarrZII= +github.com/go-openapi/swag/typeutils v0.25.5 h1:EFJ+PCga2HfHGdo8s8VJXEVbeXRCYwzzr9u4rJk7L7E= +github.com/go-openapi/swag/typeutils v0.25.5/go.mod h1:itmFmScAYE1bSD8C4rS0W+0InZUBrB2xSPbWt6DLGuc= +github.com/go-openapi/swag/yamlutils v0.25.5 h1:kASCIS+oIeoc55j28T4o8KwlV2S4ZLPT6G0iq2SSbVQ= +github.com/go-openapi/swag/yamlutils v0.25.5/go.mod h1:Gek1/SjjfbYvM+Iq4QGwa/2lEXde9n2j4a3wI3pNuOQ= +github.com/go-openapi/testify/enable/yaml/v2 v2.4.0 h1:7SgOMTvJkM8yWrQlU8Jm18VeDPuAvB/xWrdxFJkoFag= +github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7w1xVPW3CO3q1Gj04Jy//Kw4VM= +github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM= +github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= @@ -135,8 +139,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20260202012954-cb029daf43ef h1:xpF9fUHpoIrrjX24DURVKiwHcFpw19ndIs+FwTSMbno= -github.com/google/pprof v0.0.0-20260202012954-cb029daf43ef/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/pprof v0.0.0-20260302011040-a15ffb7f9dcc h1:VBbFa1lDYWEeV5FZKUiYKYT0VxCp9twUmmaq9eb8sXw= +github.com/google/pprof v0.0.0-20260302011040-a15ffb7f9dcc/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= @@ -198,8 +202,8 @@ github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTU github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos= github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= -github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= -github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -237,16 +241,18 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU= go.opentelemetry.io/otel/exporters/prometheus v0.64.0 h1:g0LRDXMX/G1SEZtK8zl8Chm4K6GBwRkjPKE36LxiTYs= go.opentelemetry.io/otel/exporters/prometheus v0.64.0/go.mod h1:UrgcjnarfdlBDP3GjDIJWe6HTprwSazNjwsI+Ru6hro= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= @@ -257,8 +263,8 @@ go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9 go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= -go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= -go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -267,53 +273,53 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= -golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o= -golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= -golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 h1:jiDhWWeC7jfWqR9c/uplMOqJ0sbNlNWv0UkzE0vX1MA= +golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90/go.mod h1:xE1HEv6b+1SCZ5/uscMRjUBKtIxworgEcEi+/n9NQDQ= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= -golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= -golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= -golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= -golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= -golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= -golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= +golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20260217215200-42d3e9bedb6d h1:EocjzKLywydp5uZ5tJ79iP6Q0UjDnyiHkGRWxuPBP8s= -google.golang.org/genproto/googleapis/api v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:48U2I+QQUYhsFrg2SY6r+nJzeOtjey7j//WBESw+qyQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d h1:t/LOSXPJ9R0B6fnZNyALBRfZBH0Uy0gT+uR+SJ6syqQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= -google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/genproto/googleapis/api v0.0.0-20260311181403-84a4fc48630c h1:OyQPd6I3pN/9gDxz6L13kYGJgqkpdrAohJRBeXyxlgI= +google.golang.org/genproto/googleapis/api v0.0.0-20260311181403-84a4fc48630c/go.mod h1:X2gu9Qwng7Nn009s/r3RUxqkzQNqOrAy79bluY7ojIg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c h1:xgCzyF2LFIO/0X2UAoVRiXKU5Xg6VjToG4i2/ecSswk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= +google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -342,10 +348,10 @@ k8s.io/client-go v0.35.2 h1:YUfPefdGJA4aljDdayAXkc98DnPkIetMl4PrKX97W9o= k8s.io/client-go v0.35.2/go.mod h1:4QqEwh4oQpeK8AaefZ0jwTFJw/9kIjdQi0jpKeYvz7g= k8s.io/component-base v0.35.2 h1:btgR+qNrpWuRSuvWSnQYsZy88yf5gVwemvz0yw79pGc= k8s.io/component-base v0.35.2/go.mod h1:B1iBJjooe6xIJYUucAxb26RwhAjzx0gHnqO9htWIX+0= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4 h1:HhDfevmPS+OalTjQRKbTHppRIz01AWi8s45TMXStgYY= -k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260304202019-5b3e3fdb0acf h1:btPscg4cMql0XdYK2jLsJcNEKmACJz8l+U7geC06FiM= +k8s.io/kube-openapi v0.0.0-20260304202019-5b3e3fdb0acf/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= diff --git a/test/e2e/setup/flux-services/kustomization.yaml b/test/e2e/setup/flux/kustomization.yaml similarity index 89% rename from test/e2e/setup/flux-services/kustomization.yaml rename to test/e2e/setup/flux/kustomization.yaml index 495ec005b..e5d63b5f7 100644 --- a/test/e2e/setup/flux-services/kustomization.yaml +++ b/test/e2e/setup/flux/kustomization.yaml @@ -1,11 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -labels: - - includeSelectors: false - pairs: - e2e.configbutler.io/stack: services - resources: - namespaces/cert-manager.yaml - namespaces/gitea-e2e.yaml diff --git a/test/e2e/setup/flux-services/namespaces/cert-manager.yaml b/test/e2e/setup/flux/namespaces/cert-manager.yaml similarity index 100% rename from test/e2e/setup/flux-services/namespaces/cert-manager.yaml rename to test/e2e/setup/flux/namespaces/cert-manager.yaml diff --git a/test/e2e/setup/flux-services/namespaces/gitea-e2e.yaml b/test/e2e/setup/flux/namespaces/gitea-e2e.yaml similarity index 100% rename from test/e2e/setup/flux-services/namespaces/gitea-e2e.yaml rename to test/e2e/setup/flux/namespaces/gitea-e2e.yaml diff --git a/test/e2e/setup/flux-services/namespaces/prometheus-operator.yaml b/test/e2e/setup/flux/namespaces/prometheus-operator.yaml similarity index 100% rename from test/e2e/setup/flux-services/namespaces/prometheus-operator.yaml rename to test/e2e/setup/flux/namespaces/prometheus-operator.yaml diff --git a/test/e2e/setup/flux-services/namespaces/valkey-e2e.yaml b/test/e2e/setup/flux/namespaces/valkey-e2e.yaml similarity index 100% rename from test/e2e/setup/flux-services/namespaces/valkey-e2e.yaml rename to test/e2e/setup/flux/namespaces/valkey-e2e.yaml diff --git a/test/e2e/setup/flux-services/releases/cert-manager.yaml b/test/e2e/setup/flux/releases/cert-manager.yaml similarity index 100% rename from test/e2e/setup/flux-services/releases/cert-manager.yaml rename to test/e2e/setup/flux/releases/cert-manager.yaml diff --git a/test/e2e/setup/flux-services/releases/gitea.yaml b/test/e2e/setup/flux/releases/gitea.yaml similarity index 100% rename from test/e2e/setup/flux-services/releases/gitea.yaml rename to test/e2e/setup/flux/releases/gitea.yaml diff --git a/test/e2e/setup/flux-services/releases/prometheus-operator.yaml b/test/e2e/setup/flux/releases/prometheus-operator.yaml similarity index 100% rename from test/e2e/setup/flux-services/releases/prometheus-operator.yaml rename to test/e2e/setup/flux/releases/prometheus-operator.yaml diff --git a/test/e2e/setup/flux-services/releases/valkey.yaml b/test/e2e/setup/flux/releases/valkey.yaml similarity index 100% rename from test/e2e/setup/flux-services/releases/valkey.yaml rename to test/e2e/setup/flux/releases/valkey.yaml diff --git a/test/e2e/setup/flux-services/repositories/gitea-charts.yaml b/test/e2e/setup/flux/repositories/gitea-charts.yaml similarity index 100% rename from test/e2e/setup/flux-services/repositories/gitea-charts.yaml rename to test/e2e/setup/flux/repositories/gitea-charts.yaml diff --git a/test/e2e/setup/flux-services/repositories/jetstack.yaml b/test/e2e/setup/flux/repositories/jetstack.yaml similarity index 100% rename from test/e2e/setup/flux-services/repositories/jetstack.yaml rename to test/e2e/setup/flux/repositories/jetstack.yaml diff --git a/test/e2e/setup/flux-services/repositories/prometheus-operator.yaml b/test/e2e/setup/flux/repositories/prometheus-operator.yaml similarity index 100% rename from test/e2e/setup/flux-services/repositories/prometheus-operator.yaml rename to test/e2e/setup/flux/repositories/prometheus-operator.yaml diff --git a/test/e2e/setup/flux-services/repositories/valkey.yaml b/test/e2e/setup/flux/repositories/valkey.yaml similarity index 100% rename from test/e2e/setup/flux-services/repositories/valkey.yaml rename to test/e2e/setup/flux/repositories/valkey.yaml diff --git a/test/e2e/setup/flux-services/values/gitea-values.yaml b/test/e2e/setup/flux/values/gitea-values.yaml similarity index 100% rename from test/e2e/setup/flux-services/values/gitea-values.yaml rename to test/e2e/setup/flux/values/gitea-values.yaml diff --git a/test/e2e/setup/flux-services/values/valkey-values.yaml b/test/e2e/setup/flux/values/valkey-values.yaml similarity index 100% rename from test/e2e/setup/flux-services/values/valkey-values.yaml rename to test/e2e/setup/flux/values/valkey-values.yaml From 3b27c8113ca0fc76348c5c2961c54f1ae251f869 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Fri, 13 Mar 2026 12:05:42 +0000 Subject: [PATCH 8/8] chore: Let's make it a bit more silent for now --- Makefile | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index e32102ff7..48e8ef1c9 100644 --- a/Makefile +++ b/Makefile @@ -337,27 +337,33 @@ $(CS)/flux.installed: $(CS)/ready | $(CS) $(FLUX) version --client > $@ # Aggregate stamp for Flux-managed external E2E services required by tests. +.SILENT: $(CS)/flux-setup.ready $(CS)/services.ready $(CS)/flux-setup.ready: $(FLUX_SETUP_READY_INPUTS) | $(CS) kubectl --context $(CTX) apply -k $(FLUX_SERVICES_DIR) - flux_ready_count=0; \ + flux_ready_count=0 + echo "⏳ Waiting for Flux-managed installations to become ready..." for kind in \ helmreleases.helm.toolkit.fluxcd.io \ - kustomizations.kustomize.toolkit.fluxcd.io \ - ; do \ - resources="$$(kubectl --context $(CTX) get $$kind --all-namespaces -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' 2>/dev/null)"; \ - [ -z "$$resources" ] && continue; \ - flux_ready_count="$$(($$flux_ready_count + $$(printf '%s\n' "$$resources" | sed '/^$$/d' | wc -l | tr -d ' ')))"; \ - printf '%s\n' "$$resources" | while read -r namespace name; do \ - [ -n "$$namespace" ] || continue; \ - kubectl --context $(CTX) -n "$$namespace" wait "$$kind/$$name" --for=condition=Ready --timeout=$(FLUX_SERVICES_WAIT_TIMEOUT); \ - done; \ - done; \ + kustomizations.kustomize.toolkit.fluxcd.io + do + resources="$$(kubectl --context $(CTX) get $$kind --all-namespaces -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' 2>/dev/null)" + [ -z "$$resources" ] && continue + + resource_count="$$(printf '%s\n' "$$resources" | sed '/^$$/d' | wc -l | tr -d ' ')" + flux_ready_count="$$(($$flux_ready_count + $$resource_count))" + + printf '%s\n' "$$resources" | while read -r namespace name; do + [ -n "$$namespace" ] || continue + kubectl --context $(CTX) -n "$$namespace" wait "$$kind/$$name" --for=condition=Ready --timeout=$(FLUX_SERVICES_WAIT_TIMEOUT) + done + done [ "$$flux_ready_count" -gt 0 ] || { echo "ERROR: no Flux-managed e2e ready-check resources found" >&2; exit 1; } - kubectl --context $(CTX) wait --for=condition=Established crd/prometheuses.monitoring.coreos.com crd/servicemonitors.monitoring.coreos.com --timeout=180s + echo "✓ Flux-managed installations ready: $$flux_ready_count" touch $@ # Aggregate stamp for external E2E services required by tests. $(CS)/services.ready: $(SERVICES_READY_INPUTS) | $(CS) + kubectl --context $(CTX) wait --for=condition=Established crd/prometheuses.monitoring.coreos.com crd/servicemonitors.monitoring.coreos.com --timeout=180s kubectl --context $(CTX) apply -R -f $(LAST_STEP_MANIFESTS_DIR) touch $@